2024-11-11 16:37:29,526 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-11 16:37:29,559 main DEBUG Took 0.010440 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 16:37:29,559 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 16:37:29,559 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 16:37:29,560 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 16:37:29,561 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:37:29,569 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 16:37:29,581 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:37:29,582 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:37:29,583 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:37:29,583 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:37:29,584 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:37:29,584 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:37:29,585 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:37:29,585 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:37:29,585 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:37:29,586 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:37:29,586 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:37:29,587 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:37:29,587 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:37:29,587 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:37:29,588 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:37:29,588 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:37:29,588 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:37:29,589 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:37:29,589 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:37:29,589 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:37:29,590 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:37:29,590 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:37:29,591 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:37:29,591 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:37:29,591 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:37:29,591 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 16:37:29,593 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:37:29,594 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 16:37:29,596 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 16:37:29,596 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 16:37:29,597 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 16:37:29,597 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 16:37:29,606 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 16:37:29,608 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 16:37:29,610 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 16:37:29,610 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 16:37:29,610 main DEBUG createAppenders(={Console}) 2024-11-11 16:37:29,611 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b initialized 2024-11-11 16:37:29,611 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-11 16:37:29,611 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b OK. 2024-11-11 16:37:29,612 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 16:37:29,612 main DEBUG OutputStream closed 2024-11-11 16:37:29,612 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 16:37:29,613 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 16:37:29,613 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@37d4349f OK 2024-11-11 16:37:29,682 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 16:37:29,684 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 16:37:29,685 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 16:37:29,686 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 16:37:29,687 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 16:37:29,687 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 16:37:29,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 16:37:29,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 16:37:29,689 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 16:37:29,689 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 16:37:29,690 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 16:37:29,690 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 16:37:29,691 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 16:37:29,691 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 16:37:29,691 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 16:37:29,692 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 16:37:29,692 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 16:37:29,693 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 16:37:29,695 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 16:37:29,695 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4331d187) with optional ClassLoader: null 2024-11-11 16:37:29,696 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 16:37:29,697 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4331d187] started OK. 2024-11-11T16:37:29,722 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync timeout: 13 mins 2024-11-11 16:37:29,726 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 16:37:29,727 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T16:37:29,802 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=11, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=439, ProcessCount=11, AvailableMemoryMB=3045 2024-11-11T16:37:30,135 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040 2024-11-11T16:37:30,812 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T16:37:30,911 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T16:37:30,965 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=wal, suffix=, logDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync, archiveDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/oldWALs, maxLogs=55 2024-11-11T16:37:31,021 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051010 2024-11-11T16:37:31,096 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051010 2024-11-11T16:37:31,177 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,184 INFO [Time-limited test {}] regionserver.ChunkCreator(472): data poolSizePercentage is less than 0. So not using pool 2024-11-11T16:37:31,184 INFO [Time-limited test {}] regionserver.ChunkCreator(472): index poolSizePercentage is less than 0. So not using pool 2024-11-11T16:37:31,223 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 682720dead0ade45c99f3f8b05a66179, NAME => 'testLockupAroundBadAssignSync,,1731343051198.682720dead0ade45c99f3f8b05a66179.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testLockupAroundBadAssignSync', {TABLE_ATTRIBUTES => {DURABILITY => 'SYNC_WAL', READONLY => 'false'}}, {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}}, regionDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040 2024-11-11T16:37:31,275 INFO [Time-limited test {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T16:37:31,281 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testLockupAroundBadAssignSync,,1731343051198.682720dead0ade45c99f3f8b05a66179.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:37:31,378 INFO [StoreOpener-682720dead0ade45c99f3f8b05a66179-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family MyCF of region 682720dead0ade45c99f3f8b05a66179 2024-11-11T16:37:31,432 INFO [StoreOpener-682720dead0ade45c99f3f8b05a66179-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 682720dead0ade45c99f3f8b05a66179 columnFamilyName MyCF 2024-11-11T16:37:31,439 DEBUG [StoreOpener-682720dead0ade45c99f3f8b05a66179-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:37:31,445 INFO [StoreOpener-682720dead0ade45c99f3f8b05a66179-1 {}] regionserver.HStore(327): Store=682720dead0ade45c99f3f8b05a66179/MyCF, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:37:31,449 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 682720dead0ade45c99f3f8b05a66179 2024-11-11T16:37:31,454 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/data/default/testLockupAroundBadAssignSync/682720dead0ade45c99f3f8b05a66179 2024-11-11T16:37:31,455 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/data/default/testLockupAroundBadAssignSync/682720dead0ade45c99f3f8b05a66179 2024-11-11T16:37:31,457 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 682720dead0ade45c99f3f8b05a66179 2024-11-11T16:37:31,458 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 682720dead0ade45c99f3f8b05a66179 2024-11-11T16:37:31,464 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 682720dead0ade45c99f3f8b05a66179 2024-11-11T16:37:31,475 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/data/default/testLockupAroundBadAssignSync/682720dead0ade45c99f3f8b05a66179/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:37:31,477 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 682720dead0ade45c99f3f8b05a66179; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72157439, jitterRate=0.07522962987422943}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T16:37:31,484 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T16:37:31,496 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 682720dead0ade45c99f3f8b05a66179: Writing region info on filesystem at 1731343051335Initializing all the Stores at 1731343051336 (+1 ms)Instantiating store for column family {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}} at 1731343051337 (+1 ms)Cleaning up temporary data from old regions at 1731343051458 (+121 ms)Region opened successfully at 1731343051494 (+36 ms) 2024-11-11T16:37:31,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051010) roll requested 2024-11-11T16:37:31,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051547 2024-11-11T16:37:31,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051010 with entries=1, filesize=249 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051547 2024-11-11T16:37:31,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051547) roll requested 2024-11-11T16:37:31,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051562 2024-11-11T16:37:31,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051562 2024-11-11T16:37:31,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051562) roll requested 2024-11-11T16:37:31,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051574 2024-11-11T16:37:31,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051574 2024-11-11T16:37:31,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051574) roll requested 2024-11-11T16:37:31,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051586 2024-11-11T16:37:31,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051586 2024-11-11T16:37:31,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051586) roll requested 2024-11-11T16:37:31,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051597 2024-11-11T16:37:31,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051597 2024-11-11T16:37:31,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051597) roll requested 2024-11-11T16:37:31,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051609 2024-11-11T16:37:31,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051609 2024-11-11T16:37:31,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051609) roll requested 2024-11-11T16:37:31,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051619 2024-11-11T16:37:31,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051619 2024-11-11T16:37:31,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051619) roll requested 2024-11-11T16:37:31,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051630 2024-11-11T16:37:31,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051630 2024-11-11T16:37:31,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051630) roll requested 2024-11-11T16:37:31,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051641 2024-11-11T16:37:31,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051641 2024-11-11T16:37:31,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051641) roll requested 2024-11-11T16:37:31,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051653 2024-11-11T16:37:31,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051653 2024-11-11T16:37:31,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051653) roll requested 2024-11-11T16:37:31,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051663 2024-11-11T16:37:31,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051663 2024-11-11T16:37:31,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051663) roll requested 2024-11-11T16:37:31,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051674 2024-11-11T16:37:31,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051674 2024-11-11T16:37:31,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051674) roll requested 2024-11-11T16:37:31,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051684 2024-11-11T16:37:31,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051684 2024-11-11T16:37:31,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051684) roll requested 2024-11-11T16:37:31,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051694 2024-11-11T16:37:31,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051694 2024-11-11T16:37:31,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051694) roll requested 2024-11-11T16:37:31,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051704 2024-11-11T16:37:31,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051704 2024-11-11T16:37:31,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051704) roll requested 2024-11-11T16:37:31,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051715 2024-11-11T16:37:31,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051715 2024-11-11T16:37:31,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051715) roll requested 2024-11-11T16:37:31,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051725 2024-11-11T16:37:31,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051725 2024-11-11T16:37:31,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051725) roll requested 2024-11-11T16:37:31,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051737 2024-11-11T16:37:31,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051737 2024-11-11T16:37:31,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051737) roll requested 2024-11-11T16:37:31,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051748 2024-11-11T16:37:31,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051748 2024-11-11T16:37:31,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051748) roll requested 2024-11-11T16:37:31,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051757 2024-11-11T16:37:31,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051757 2024-11-11T16:37:31,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051757) roll requested 2024-11-11T16:37:31,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051767 2024-11-11T16:37:31,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051767 2024-11-11T16:37:31,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051767) roll requested 2024-11-11T16:37:31,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051776 2024-11-11T16:37:31,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051776 2024-11-11T16:37:31,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051776) roll requested 2024-11-11T16:37:31,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051789 2024-11-11T16:37:31,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051789 2024-11-11T16:37:31,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051789) roll requested 2024-11-11T16:37:31,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051804 2024-11-11T16:37:31,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051804 2024-11-11T16:37:31,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051804) roll requested 2024-11-11T16:37:31,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051821 2024-11-11T16:37:31,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051821 2024-11-11T16:37:31,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051821) roll requested 2024-11-11T16:37:31,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051829 2024-11-11T16:37:31,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051829 2024-11-11T16:37:31,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051829) roll requested 2024-11-11T16:37:31,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051845 2024-11-11T16:37:31,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051845 2024-11-11T16:37:31,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051845) roll requested 2024-11-11T16:37:31,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051860 2024-11-11T16:37:31,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051860 2024-11-11T16:37:31,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051860) roll requested 2024-11-11T16:37:31,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051872 2024-11-11T16:37:31,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051872 2024-11-11T16:37:31,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051872) roll requested 2024-11-11T16:37:31,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051884 2024-11-11T16:37:31,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051884 2024-11-11T16:37:31,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051884) roll requested 2024-11-11T16:37:31,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051894 2024-11-11T16:37:31,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051894 2024-11-11T16:37:31,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051894) roll requested 2024-11-11T16:37:31,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051906 2024-11-11T16:37:31,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051906 2024-11-11T16:37:31,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051906) roll requested 2024-11-11T16:37:31,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051915 2024-11-11T16:37:31,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051915 2024-11-11T16:37:31,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051915) roll requested 2024-11-11T16:37:31,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051925 2024-11-11T16:37:31,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051925 2024-11-11T16:37:31,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051925) roll requested 2024-11-11T16:37:31,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051934 2024-11-11T16:37:31,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051934 2024-11-11T16:37:31,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051934) roll requested 2024-11-11T16:37:31,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051945 2024-11-11T16:37:31,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051945 2024-11-11T16:37:31,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051945) roll requested 2024-11-11T16:37:31,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051955 2024-11-11T16:37:31,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051955 2024-11-11T16:37:31,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051955) roll requested 2024-11-11T16:37:31,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051963 2024-11-11T16:37:31,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051963 2024-11-11T16:37:31,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051963) roll requested 2024-11-11T16:37:31,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051973 2024-11-11T16:37:31,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051973 2024-11-11T16:37:31,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051973) roll requested 2024-11-11T16:37:31,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051985 2024-11-11T16:37:31,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:31,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051985 2024-11-11T16:37:31,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:31,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:31,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051985) roll requested 2024-11-11T16:37:31,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343051994 2024-11-11T16:37:32,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051994 2024-11-11T16:37:32,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343051994) roll requested 2024-11-11T16:37:32,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052002 2024-11-11T16:37:32,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343051994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052002 2024-11-11T16:37:32,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052002) roll requested 2024-11-11T16:37:32,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052012 2024-11-11T16:37:32,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052012 2024-11-11T16:37:32,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052012) roll requested 2024-11-11T16:37:32,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052022 2024-11-11T16:37:32,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052022 2024-11-11T16:37:32,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052022) roll requested 2024-11-11T16:37:32,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052032 2024-11-11T16:37:32,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052032 2024-11-11T16:37:32,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052032) roll requested 2024-11-11T16:37:32,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052041 2024-11-11T16:37:32,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052041 2024-11-11T16:37:32,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052041) roll requested 2024-11-11T16:37:32,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052051 2024-11-11T16:37:32,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052051 2024-11-11T16:37:32,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052051) roll requested 2024-11-11T16:37:32,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052059 2024-11-11T16:37:32,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052059 2024-11-11T16:37:32,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052059) roll requested 2024-11-11T16:37:32,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052069 2024-11-11T16:37:32,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052069 2024-11-11T16:37:32,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052069) roll requested 2024-11-11T16:37:32,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052081 2024-11-11T16:37:32,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052081 2024-11-11T16:37:32,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052081) roll requested 2024-11-11T16:37:32,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052097 2024-11-11T16:37:32,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052097 2024-11-11T16:37:32,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052097) roll requested 2024-11-11T16:37:32,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052106 2024-11-11T16:37:32,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052106 2024-11-11T16:37:32,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052106) roll requested 2024-11-11T16:37:32,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052117 2024-11-11T16:37:32,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052117 2024-11-11T16:37:32,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052117) roll requested 2024-11-11T16:37:32,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052130 2024-11-11T16:37:32,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052130 2024-11-11T16:37:32,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052130) roll requested 2024-11-11T16:37:32,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052141 2024-11-11T16:37:32,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052141 2024-11-11T16:37:32,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052141) roll requested 2024-11-11T16:37:32,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052150 2024-11-11T16:37:32,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052150 2024-11-11T16:37:32,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052150) roll requested 2024-11-11T16:37:32,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052160 2024-11-11T16:37:32,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052160 2024-11-11T16:37:32,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052160) roll requested 2024-11-11T16:37:32,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052169 2024-11-11T16:37:32,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052169 2024-11-11T16:37:32,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052169) roll requested 2024-11-11T16:37:32,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052177 2024-11-11T16:37:32,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052177 2024-11-11T16:37:32,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052177) roll requested 2024-11-11T16:37:32,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052185 2024-11-11T16:37:32,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052185 2024-11-11T16:37:32,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052185) roll requested 2024-11-11T16:37:32,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052194 2024-11-11T16:37:32,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052194 2024-11-11T16:37:32,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052194) roll requested 2024-11-11T16:37:32,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052202 2024-11-11T16:37:32,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052202 2024-11-11T16:37:32,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052202) roll requested 2024-11-11T16:37:32,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052210 2024-11-11T16:37:32,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052210 2024-11-11T16:37:32,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052210) roll requested 2024-11-11T16:37:32,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052217 2024-11-11T16:37:32,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052217 2024-11-11T16:37:32,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052217) roll requested 2024-11-11T16:37:32,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052225 2024-11-11T16:37:32,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052225 2024-11-11T16:37:32,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052225) roll requested 2024-11-11T16:37:32,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052234 2024-11-11T16:37:32,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052234 2024-11-11T16:37:32,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052234) roll requested 2024-11-11T16:37:32,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052243 2024-11-11T16:37:32,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052243 2024-11-11T16:37:32,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052243) roll requested 2024-11-11T16:37:32,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052251 2024-11-11T16:37:32,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052251 2024-11-11T16:37:32,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052251) roll requested 2024-11-11T16:37:32,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052260 2024-11-11T16:37:32,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052260 2024-11-11T16:37:32,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052260) roll requested 2024-11-11T16:37:32,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052268 2024-11-11T16:37:32,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052268 2024-11-11T16:37:32,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052268) roll requested 2024-11-11T16:37:32,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052277 2024-11-11T16:37:32,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052277 2024-11-11T16:37:32,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052277) roll requested 2024-11-11T16:37:32,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052285 2024-11-11T16:37:32,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052285 2024-11-11T16:37:32,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052285) roll requested 2024-11-11T16:37:32,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052292 2024-11-11T16:37:32,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052292 2024-11-11T16:37:32,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052292) roll requested 2024-11-11T16:37:32,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052300 2024-11-11T16:37:32,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052300 2024-11-11T16:37:32,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052300) roll requested 2024-11-11T16:37:32,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052308 2024-11-11T16:37:32,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052308 2024-11-11T16:37:32,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052308) roll requested 2024-11-11T16:37:32,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052318 2024-11-11T16:37:32,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052318 2024-11-11T16:37:32,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052318) roll requested 2024-11-11T16:37:32,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052326 2024-11-11T16:37:32,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052326 2024-11-11T16:37:32,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052326) roll requested 2024-11-11T16:37:32,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052334 2024-11-11T16:37:32,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052334 2024-11-11T16:37:32,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052334) roll requested 2024-11-11T16:37:32,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052342 2024-11-11T16:37:32,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052342 2024-11-11T16:37:32,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052342) roll requested 2024-11-11T16:37:32,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052350 2024-11-11T16:37:32,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052350 2024-11-11T16:37:32,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052350) roll requested 2024-11-11T16:37:32,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052359 2024-11-11T16:37:32,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052359 2024-11-11T16:37:32,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052359) roll requested 2024-11-11T16:37:32,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052366 2024-11-11T16:37:32,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052366 2024-11-11T16:37:32,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052366) roll requested 2024-11-11T16:37:32,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052374 2024-11-11T16:37:32,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052374 2024-11-11T16:37:32,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052374) roll requested 2024-11-11T16:37:32,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052381 2024-11-11T16:37:32,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052381 2024-11-11T16:37:32,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052381) roll requested 2024-11-11T16:37:32,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052389 2024-11-11T16:37:32,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052389 2024-11-11T16:37:32,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052389) roll requested 2024-11-11T16:37:32,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052397 2024-11-11T16:37:32,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052397 2024-11-11T16:37:32,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052397) roll requested 2024-11-11T16:37:32,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052408 2024-11-11T16:37:32,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052408 2024-11-11T16:37:32,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052408) roll requested 2024-11-11T16:37:32,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052416 2024-11-11T16:37:32,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052416 2024-11-11T16:37:32,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052416) roll requested 2024-11-11T16:37:32,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052424 2024-11-11T16:37:32,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052424 2024-11-11T16:37:32,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052424) roll requested 2024-11-11T16:37:32,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052436 2024-11-11T16:37:32,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052436 2024-11-11T16:37:32,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052436) roll requested 2024-11-11T16:37:32,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052447 2024-11-11T16:37:32,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052447 2024-11-11T16:37:32,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052447) roll requested 2024-11-11T16:37:32,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052456 2024-11-11T16:37:32,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052456 2024-11-11T16:37:32,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052456) roll requested 2024-11-11T16:37:32,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052468 2024-11-11T16:37:32,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052468 2024-11-11T16:37:32,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052468) roll requested 2024-11-11T16:37:32,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052483 2024-11-11T16:37:32,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052483 2024-11-11T16:37:32,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052483) roll requested 2024-11-11T16:37:32,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052494 2024-11-11T16:37:32,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052494 2024-11-11T16:37:32,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052494) roll requested 2024-11-11T16:37:32,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052513 2024-11-11T16:37:32,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052513 2024-11-11T16:37:32,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052513) roll requested 2024-11-11T16:37:32,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052524 2024-11-11T16:37:32,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052524 2024-11-11T16:37:32,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052524) roll requested 2024-11-11T16:37:32,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052534 2024-11-11T16:37:32,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052534 2024-11-11T16:37:32,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052534) roll requested 2024-11-11T16:37:32,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052544 2024-11-11T16:37:32,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052544 2024-11-11T16:37:32,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052544) roll requested 2024-11-11T16:37:32,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052557 2024-11-11T16:37:32,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052557 2024-11-11T16:37:32,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052557) roll requested 2024-11-11T16:37:32,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052565 2024-11-11T16:37:32,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052565 2024-11-11T16:37:32,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052565) roll requested 2024-11-11T16:37:32,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052583 2024-11-11T16:37:32,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052583 2024-11-11T16:37:32,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052583) roll requested 2024-11-11T16:37:32,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052593 2024-11-11T16:37:32,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052593 2024-11-11T16:37:32,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052593) roll requested 2024-11-11T16:37:32,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052603 2024-11-11T16:37:32,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052603 2024-11-11T16:37:32,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052603) roll requested 2024-11-11T16:37:32,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052614 2024-11-11T16:37:32,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052614 2024-11-11T16:37:32,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052614) roll requested 2024-11-11T16:37:32,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052628 2024-11-11T16:37:32,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052628 2024-11-11T16:37:32,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052628) roll requested 2024-11-11T16:37:32,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052640 2024-11-11T16:37:32,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052640 2024-11-11T16:37:32,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052640) roll requested 2024-11-11T16:37:32,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052651 2024-11-11T16:37:32,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052651 2024-11-11T16:37:32,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052651) roll requested 2024-11-11T16:37:32,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052660 2024-11-11T16:37:32,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052660 2024-11-11T16:37:32,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052660) roll requested 2024-11-11T16:37:32,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052668 2024-11-11T16:37:32,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052668 2024-11-11T16:37:32,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052668) roll requested 2024-11-11T16:37:32,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052676 2024-11-11T16:37:32,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052676 2024-11-11T16:37:32,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052676) roll requested 2024-11-11T16:37:32,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052684 2024-11-11T16:37:32,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052684 2024-11-11T16:37:32,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052684) roll requested 2024-11-11T16:37:32,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052692 2024-11-11T16:37:32,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052692 2024-11-11T16:37:32,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052692) roll requested 2024-11-11T16:37:32,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052703 2024-11-11T16:37:32,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052703 2024-11-11T16:37:32,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052703) roll requested 2024-11-11T16:37:32,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052714 2024-11-11T16:37:32,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052714 2024-11-11T16:37:32,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052714) roll requested 2024-11-11T16:37:32,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052722 2024-11-11T16:37:32,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052722 2024-11-11T16:37:32,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052722) roll requested 2024-11-11T16:37:32,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052734 2024-11-11T16:37:32,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052734 2024-11-11T16:37:32,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052734) roll requested 2024-11-11T16:37:32,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052743 2024-11-11T16:37:32,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052743 2024-11-11T16:37:32,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052743) roll requested 2024-11-11T16:37:32,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052751 2024-11-11T16:37:32,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052751 2024-11-11T16:37:32,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052751) roll requested 2024-11-11T16:37:32,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052759 2024-11-11T16:37:32,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052759 2024-11-11T16:37:32,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052759) roll requested 2024-11-11T16:37:32,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052767 2024-11-11T16:37:32,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052767 2024-11-11T16:37:32,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052767) roll requested 2024-11-11T16:37:32,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052774 2024-11-11T16:37:32,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052774 2024-11-11T16:37:32,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052774) roll requested 2024-11-11T16:37:32,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052783 2024-11-11T16:37:32,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052783 2024-11-11T16:37:32,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052783) roll requested 2024-11-11T16:37:32,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052791 2024-11-11T16:37:32,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052791 2024-11-11T16:37:32,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052791) roll requested 2024-11-11T16:37:32,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052799 2024-11-11T16:37:32,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052799 2024-11-11T16:37:32,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052799) roll requested 2024-11-11T16:37:32,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052806 2024-11-11T16:37:32,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052806 2024-11-11T16:37:32,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052806) roll requested 2024-11-11T16:37:32,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052814 2024-11-11T16:37:32,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052814 2024-11-11T16:37:32,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052814) roll requested 2024-11-11T16:37:32,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052821 2024-11-11T16:37:32,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052821 2024-11-11T16:37:32,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052821) roll requested 2024-11-11T16:37:32,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052829 2024-11-11T16:37:32,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052829 2024-11-11T16:37:32,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052829) roll requested 2024-11-11T16:37:32,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052836 2024-11-11T16:37:32,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052836 2024-11-11T16:37:32,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052836) roll requested 2024-11-11T16:37:32,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052844 2024-11-11T16:37:32,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052844 2024-11-11T16:37:32,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052844) roll requested 2024-11-11T16:37:32,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052856 2024-11-11T16:37:32,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052856 2024-11-11T16:37:32,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052856) roll requested 2024-11-11T16:37:32,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052863 2024-11-11T16:37:32,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052863 2024-11-11T16:37:32,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052863) roll requested 2024-11-11T16:37:32,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052870 2024-11-11T16:37:32,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052870 2024-11-11T16:37:32,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052870) roll requested 2024-11-11T16:37:32,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052878 2024-11-11T16:37:32,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052878 2024-11-11T16:37:32,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052878) roll requested 2024-11-11T16:37:32,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052885 2024-11-11T16:37:32,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052885 2024-11-11T16:37:32,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052885) roll requested 2024-11-11T16:37:32,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052893 2024-11-11T16:37:32,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052893 2024-11-11T16:37:32,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052893) roll requested 2024-11-11T16:37:32,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052901 2024-11-11T16:37:32,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052901 2024-11-11T16:37:32,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052901) roll requested 2024-11-11T16:37:32,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052909 2024-11-11T16:37:32,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052909 2024-11-11T16:37:32,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052909) roll requested 2024-11-11T16:37:32,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052916 2024-11-11T16:37:32,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052916 2024-11-11T16:37:32,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052916) roll requested 2024-11-11T16:37:32,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052924 2024-11-11T16:37:32,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052924 2024-11-11T16:37:32,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052924) roll requested 2024-11-11T16:37:32,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052932 2024-11-11T16:37:32,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052932 2024-11-11T16:37:32,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052932) roll requested 2024-11-11T16:37:32,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052940 2024-11-11T16:37:32,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052940 2024-11-11T16:37:32,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052940) roll requested 2024-11-11T16:37:32,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052953 2024-11-11T16:37:32,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052953 2024-11-11T16:37:32,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052953) roll requested 2024-11-11T16:37:32,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052965 2024-11-11T16:37:32,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052965 2024-11-11T16:37:32,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052965) roll requested 2024-11-11T16:37:32,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052975 2024-11-11T16:37:32,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052975 2024-11-11T16:37:32,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052975) roll requested 2024-11-11T16:37:32,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052984 2024-11-11T16:37:32,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052984 2024-11-11T16:37:32,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:32,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:32,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052984) roll requested 2024-11-11T16:37:32,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343052993 2024-11-11T16:37:32,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:32,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052993 2024-11-11T16:37:33,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343052993) roll requested 2024-11-11T16:37:33,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053001 2024-11-11T16:37:33,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343052993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053001 2024-11-11T16:37:33,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053001) roll requested 2024-11-11T16:37:33,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053009 2024-11-11T16:37:33,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053009 2024-11-11T16:37:33,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053009) roll requested 2024-11-11T16:37:33,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053017 2024-11-11T16:37:33,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053017 2024-11-11T16:37:33,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053017) roll requested 2024-11-11T16:37:33,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053025 2024-11-11T16:37:33,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053025 2024-11-11T16:37:33,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053025) roll requested 2024-11-11T16:37:33,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053033 2024-11-11T16:37:33,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053033 2024-11-11T16:37:33,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053033) roll requested 2024-11-11T16:37:33,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053042 2024-11-11T16:37:33,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053042 2024-11-11T16:37:33,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053042) roll requested 2024-11-11T16:37:33,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053053 2024-11-11T16:37:33,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053053 2024-11-11T16:37:33,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053053) roll requested 2024-11-11T16:37:33,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053061 2024-11-11T16:37:33,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053061 2024-11-11T16:37:33,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053061) roll requested 2024-11-11T16:37:33,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053069 2024-11-11T16:37:33,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053069 2024-11-11T16:37:33,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053069) roll requested 2024-11-11T16:37:33,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053081 2024-11-11T16:37:33,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053081 2024-11-11T16:37:33,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053081) roll requested 2024-11-11T16:37:33,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053089 2024-11-11T16:37:33,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053089 2024-11-11T16:37:33,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053089) roll requested 2024-11-11T16:37:33,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053097 2024-11-11T16:37:33,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053097 2024-11-11T16:37:33,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053097) roll requested 2024-11-11T16:37:33,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053105 2024-11-11T16:37:33,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053105 2024-11-11T16:37:33,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053105) roll requested 2024-11-11T16:37:33,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053113 2024-11-11T16:37:33,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053113 2024-11-11T16:37:33,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053113) roll requested 2024-11-11T16:37:33,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053122 2024-11-11T16:37:33,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053122 2024-11-11T16:37:33,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053122) roll requested 2024-11-11T16:37:33,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053132 2024-11-11T16:37:33,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053132 2024-11-11T16:37:33,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053132) roll requested 2024-11-11T16:37:33,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053141 2024-11-11T16:37:33,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053141 2024-11-11T16:37:33,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053141) roll requested 2024-11-11T16:37:33,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053149 2024-11-11T16:37:33,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053149 2024-11-11T16:37:33,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053149) roll requested 2024-11-11T16:37:33,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053157 2024-11-11T16:37:33,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053157 2024-11-11T16:37:33,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053157) roll requested 2024-11-11T16:37:33,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053165 2024-11-11T16:37:33,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053165 2024-11-11T16:37:33,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053165) roll requested 2024-11-11T16:37:33,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053173 2024-11-11T16:37:33,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053173 2024-11-11T16:37:33,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053173) roll requested 2024-11-11T16:37:33,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053181 2024-11-11T16:37:33,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053181 2024-11-11T16:37:33,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053181) roll requested 2024-11-11T16:37:33,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053189 2024-11-11T16:37:33,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053189 2024-11-11T16:37:33,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053189) roll requested 2024-11-11T16:37:33,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053199 2024-11-11T16:37:33,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053199 2024-11-11T16:37:33,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053199) roll requested 2024-11-11T16:37:33,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053215 2024-11-11T16:37:33,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053215 2024-11-11T16:37:33,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053215) roll requested 2024-11-11T16:37:33,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053224 2024-11-11T16:37:33,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053224 2024-11-11T16:37:33,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053224) roll requested 2024-11-11T16:37:33,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053234 2024-11-11T16:37:33,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053234 2024-11-11T16:37:33,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053234) roll requested 2024-11-11T16:37:33,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053243 2024-11-11T16:37:33,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053243 2024-11-11T16:37:33,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053243) roll requested 2024-11-11T16:37:33,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053252 2024-11-11T16:37:33,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053252 2024-11-11T16:37:33,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053252) roll requested 2024-11-11T16:37:33,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053261 2024-11-11T16:37:33,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053261 2024-11-11T16:37:33,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053261) roll requested 2024-11-11T16:37:33,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053269 2024-11-11T16:37:33,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053269 2024-11-11T16:37:33,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053269) roll requested 2024-11-11T16:37:33,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053277 2024-11-11T16:37:33,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053277 2024-11-11T16:37:33,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053277) roll requested 2024-11-11T16:37:33,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053285 2024-11-11T16:37:33,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053285 2024-11-11T16:37:33,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053285) roll requested 2024-11-11T16:37:33,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053293 2024-11-11T16:37:33,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053293 2024-11-11T16:37:33,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053293) roll requested 2024-11-11T16:37:33,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053302 2024-11-11T16:37:33,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053302 2024-11-11T16:37:33,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053302) roll requested 2024-11-11T16:37:33,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053312 2024-11-11T16:37:33,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053312 2024-11-11T16:37:33,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053312) roll requested 2024-11-11T16:37:33,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053321 2024-11-11T16:37:33,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053321 2024-11-11T16:37:33,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053321) roll requested 2024-11-11T16:37:33,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053328 2024-11-11T16:37:33,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053328 2024-11-11T16:37:33,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053328) roll requested 2024-11-11T16:37:33,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053336 2024-11-11T16:37:33,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053336 2024-11-11T16:37:33,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053336) roll requested 2024-11-11T16:37:33,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053343 2024-11-11T16:37:33,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053343 2024-11-11T16:37:33,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053343) roll requested 2024-11-11T16:37:33,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053352 2024-11-11T16:37:33,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053352 2024-11-11T16:37:33,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053352) roll requested 2024-11-11T16:37:33,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053359 2024-11-11T16:37:33,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053359 2024-11-11T16:37:33,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053359) roll requested 2024-11-11T16:37:33,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053373 2024-11-11T16:37:33,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053373 2024-11-11T16:37:33,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053373) roll requested 2024-11-11T16:37:33,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053387 2024-11-11T16:37:33,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053387 2024-11-11T16:37:33,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053387) roll requested 2024-11-11T16:37:33,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053399 2024-11-11T16:37:33,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053399 2024-11-11T16:37:33,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053399) roll requested 2024-11-11T16:37:33,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053413 2024-11-11T16:37:33,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053413 2024-11-11T16:37:33,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053413) roll requested 2024-11-11T16:37:33,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053427 2024-11-11T16:37:33,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053427 2024-11-11T16:37:33,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053427) roll requested 2024-11-11T16:37:33,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053438 2024-11-11T16:37:33,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053438 2024-11-11T16:37:33,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053438) roll requested 2024-11-11T16:37:33,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053450 2024-11-11T16:37:33,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053450 2024-11-11T16:37:33,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053450) roll requested 2024-11-11T16:37:33,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053461 2024-11-11T16:37:33,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053461 2024-11-11T16:37:33,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053461) roll requested 2024-11-11T16:37:33,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053473 2024-11-11T16:37:33,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053473 2024-11-11T16:37:33,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053473) roll requested 2024-11-11T16:37:33,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053482 2024-11-11T16:37:33,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053482 2024-11-11T16:37:33,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053482) roll requested 2024-11-11T16:37:33,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053491 2024-11-11T16:37:33,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053491 2024-11-11T16:37:33,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053491) roll requested 2024-11-11T16:37:33,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053501 2024-11-11T16:37:33,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053501 2024-11-11T16:37:33,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053501) roll requested 2024-11-11T16:37:33,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053512 2024-11-11T16:37:33,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053512 2024-11-11T16:37:33,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053512) roll requested 2024-11-11T16:37:33,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053520 2024-11-11T16:37:33,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053520 2024-11-11T16:37:33,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053520) roll requested 2024-11-11T16:37:33,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053529 2024-11-11T16:37:33,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053529 2024-11-11T16:37:33,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053529) roll requested 2024-11-11T16:37:33,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053539 2024-11-11T16:37:33,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053539 2024-11-11T16:37:33,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053529 is not closed yet, will try archiving it next time 2024-11-11T16:37:33,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053539) roll requested 2024-11-11T16:37:33,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053559 2024-11-11T16:37:33,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053559 2024-11-11T16:37:33,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053559) roll requested 2024-11-11T16:37:33,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053574 2024-11-11T16:37:33,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053574 2024-11-11T16:37:33,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053574) roll requested 2024-11-11T16:37:33,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053587 2024-11-11T16:37:33,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053587 2024-11-11T16:37:33,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053587) roll requested 2024-11-11T16:37:33,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053597 2024-11-11T16:37:33,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053597 2024-11-11T16:37:33,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053597) roll requested 2024-11-11T16:37:33,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053608 2024-11-11T16:37:33,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053608 2024-11-11T16:37:33,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053608) roll requested 2024-11-11T16:37:33,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053617 2024-11-11T16:37:33,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053617 2024-11-11T16:37:33,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053617) roll requested 2024-11-11T16:37:33,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053630 2024-11-11T16:37:33,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053630 2024-11-11T16:37:33,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053630) roll requested 2024-11-11T16:37:33,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053644 2024-11-11T16:37:33,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053644 2024-11-11T16:37:33,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053644) roll requested 2024-11-11T16:37:33,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053669 2024-11-11T16:37:33,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053669 2024-11-11T16:37:33,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053669) roll requested 2024-11-11T16:37:33,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053686 2024-11-11T16:37:33,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053686 2024-11-11T16:37:33,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053686) roll requested 2024-11-11T16:37:33,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053717 2024-11-11T16:37:33,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053717 2024-11-11T16:37:33,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053717) roll requested 2024-11-11T16:37:33,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053735 2024-11-11T16:37:33,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053735 2024-11-11T16:37:33,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053735) roll requested 2024-11-11T16:37:33,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053770 2024-11-11T16:37:33,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053770 2024-11-11T16:37:33,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053770) roll requested 2024-11-11T16:37:33,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053782 2024-11-11T16:37:33,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053782 2024-11-11T16:37:33,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053782) roll requested 2024-11-11T16:37:33,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053792 2024-11-11T16:37:33,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053792 2024-11-11T16:37:33,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053792) roll requested 2024-11-11T16:37:33,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053799 2024-11-11T16:37:33,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053799 2024-11-11T16:37:33,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053799) roll requested 2024-11-11T16:37:33,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053807 2024-11-11T16:37:33,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053807 2024-11-11T16:37:33,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053807) roll requested 2024-11-11T16:37:33,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053814 2024-11-11T16:37:33,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053814 2024-11-11T16:37:33,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053814) roll requested 2024-11-11T16:37:33,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053821 2024-11-11T16:37:33,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053821 2024-11-11T16:37:33,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053821) roll requested 2024-11-11T16:37:33,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053829 2024-11-11T16:37:33,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053829 2024-11-11T16:37:33,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053829) roll requested 2024-11-11T16:37:33,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053836 2024-11-11T16:37:33,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053836 2024-11-11T16:37:33,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053836) roll requested 2024-11-11T16:37:33,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053844 2024-11-11T16:37:33,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053844 2024-11-11T16:37:33,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053844) roll requested 2024-11-11T16:37:33,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053854 2024-11-11T16:37:33,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053854 2024-11-11T16:37:33,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053854) roll requested 2024-11-11T16:37:33,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053866 2024-11-11T16:37:33,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053866 2024-11-11T16:37:33,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053866) roll requested 2024-11-11T16:37:33,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053875 2024-11-11T16:37:33,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053875 2024-11-11T16:37:33,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053875) roll requested 2024-11-11T16:37:33,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053889 2024-11-11T16:37:33,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053889 2024-11-11T16:37:33,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053889) roll requested 2024-11-11T16:37:33,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053899 2024-11-11T16:37:33,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053899 2024-11-11T16:37:33,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053899) roll requested 2024-11-11T16:37:33,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053909 2024-11-11T16:37:33,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053909 2024-11-11T16:37:33,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053909) roll requested 2024-11-11T16:37:33,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053927 2024-11-11T16:37:33,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053927 2024-11-11T16:37:33,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053927) roll requested 2024-11-11T16:37:33,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053950 2024-11-11T16:37:33,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053950 2024-11-11T16:37:33,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:33,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:33,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053950) roll requested 2024-11-11T16:37:33,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343053970 2024-11-11T16:37:33,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:33,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053970 2024-11-11T16:37:34,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343053970) roll requested 2024-11-11T16:37:34,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054002 2024-11-11T16:37:34,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343053970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054002 2024-11-11T16:37:34,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054002) roll requested 2024-11-11T16:37:34,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054018 2024-11-11T16:37:34,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054018 2024-11-11T16:37:34,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054018) roll requested 2024-11-11T16:37:34,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054033 2024-11-11T16:37:34,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054033 2024-11-11T16:37:34,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054033) roll requested 2024-11-11T16:37:34,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054054 2024-11-11T16:37:34,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054054 2024-11-11T16:37:34,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054054) roll requested 2024-11-11T16:37:34,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054075 2024-11-11T16:37:34,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054075 2024-11-11T16:37:34,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054075) roll requested 2024-11-11T16:37:34,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054089 2024-11-11T16:37:34,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054089 2024-11-11T16:37:34,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054089) roll requested 2024-11-11T16:37:34,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054126 2024-11-11T16:37:34,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054126 2024-11-11T16:37:34,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054126) roll requested 2024-11-11T16:37:34,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054138 2024-11-11T16:37:34,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054138 2024-11-11T16:37:34,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054138) roll requested 2024-11-11T16:37:34,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054157 2024-11-11T16:37:34,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054157 2024-11-11T16:37:34,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054157) roll requested 2024-11-11T16:37:34,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054171 2024-11-11T16:37:34,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054171 2024-11-11T16:37:34,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054171) roll requested 2024-11-11T16:37:34,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054187 2024-11-11T16:37:34,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054187 2024-11-11T16:37:34,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054187) roll requested 2024-11-11T16:37:34,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054201 2024-11-11T16:37:34,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054201 2024-11-11T16:37:34,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054201) roll requested 2024-11-11T16:37:34,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054233 2024-11-11T16:37:34,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054233 2024-11-11T16:37:34,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054233) roll requested 2024-11-11T16:37:34,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054246 2024-11-11T16:37:34,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054246 2024-11-11T16:37:34,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054246) roll requested 2024-11-11T16:37:34,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054260 2024-11-11T16:37:34,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054260 2024-11-11T16:37:34,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054260) roll requested 2024-11-11T16:37:34,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054270 2024-11-11T16:37:34,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054270 2024-11-11T16:37:34,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054270) roll requested 2024-11-11T16:37:34,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054280 2024-11-11T16:37:34,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054280 2024-11-11T16:37:34,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054280) roll requested 2024-11-11T16:37:34,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054289 2024-11-11T16:37:34,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054289 2024-11-11T16:37:34,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054289) roll requested 2024-11-11T16:37:34,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054302 2024-11-11T16:37:34,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054302 2024-11-11T16:37:34,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054302) roll requested 2024-11-11T16:37:34,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054310 2024-11-11T16:37:34,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054310 2024-11-11T16:37:34,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054310) roll requested 2024-11-11T16:37:34,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054317 2024-11-11T16:37:34,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054317 2024-11-11T16:37:34,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054317) roll requested 2024-11-11T16:37:34,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054324 2024-11-11T16:37:34,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054324 2024-11-11T16:37:34,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054324) roll requested 2024-11-11T16:37:34,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054331 2024-11-11T16:37:34,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054331 2024-11-11T16:37:34,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054331) roll requested 2024-11-11T16:37:34,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054338 2024-11-11T16:37:34,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054338 2024-11-11T16:37:34,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054338) roll requested 2024-11-11T16:37:34,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054345 2024-11-11T16:37:34,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054345 2024-11-11T16:37:34,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054345) roll requested 2024-11-11T16:37:34,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054352 2024-11-11T16:37:34,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054352 2024-11-11T16:37:34,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054352) roll requested 2024-11-11T16:37:34,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054362 2024-11-11T16:37:34,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054362 2024-11-11T16:37:34,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054362) roll requested 2024-11-11T16:37:34,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054369 2024-11-11T16:37:34,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054369 2024-11-11T16:37:34,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054369) roll requested 2024-11-11T16:37:34,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054376 2024-11-11T16:37:34,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054376 2024-11-11T16:37:34,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054376) roll requested 2024-11-11T16:37:34,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054384 2024-11-11T16:37:34,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054384 2024-11-11T16:37:34,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054384) roll requested 2024-11-11T16:37:34,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054390 2024-11-11T16:37:34,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054390 2024-11-11T16:37:34,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054390) roll requested 2024-11-11T16:37:34,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054399 2024-11-11T16:37:34,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054399 2024-11-11T16:37:34,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054399) roll requested 2024-11-11T16:37:34,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054406 2024-11-11T16:37:34,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054406 2024-11-11T16:37:34,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054406) roll requested 2024-11-11T16:37:34,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054413 2024-11-11T16:37:34,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054413 2024-11-11T16:37:34,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054413) roll requested 2024-11-11T16:37:34,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054420 2024-11-11T16:37:34,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054420 2024-11-11T16:37:34,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054420) roll requested 2024-11-11T16:37:34,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054427 2024-11-11T16:37:34,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054427 2024-11-11T16:37:34,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054427) roll requested 2024-11-11T16:37:34,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054434 2024-11-11T16:37:34,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054434 2024-11-11T16:37:34,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054434) roll requested 2024-11-11T16:37:34,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054441 2024-11-11T16:37:34,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054441 2024-11-11T16:37:34,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054441) roll requested 2024-11-11T16:37:34,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054448 2024-11-11T16:37:34,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054448 2024-11-11T16:37:34,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054448) roll requested 2024-11-11T16:37:34,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054455 2024-11-11T16:37:34,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054455 2024-11-11T16:37:34,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054455) roll requested 2024-11-11T16:37:34,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054463 2024-11-11T16:37:34,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054463 2024-11-11T16:37:34,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054463) roll requested 2024-11-11T16:37:34,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054470 2024-11-11T16:37:34,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054470 2024-11-11T16:37:34,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054470) roll requested 2024-11-11T16:37:34,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054477 2024-11-11T16:37:34,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054477 2024-11-11T16:37:34,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054477) roll requested 2024-11-11T16:37:34,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054489 2024-11-11T16:37:34,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054489 2024-11-11T16:37:34,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054489) roll requested 2024-11-11T16:37:34,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054496 2024-11-11T16:37:34,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054496 2024-11-11T16:37:34,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054496) roll requested 2024-11-11T16:37:34,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054503 2024-11-11T16:37:34,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054503 2024-11-11T16:37:34,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054503) roll requested 2024-11-11T16:37:34,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054510 2024-11-11T16:37:34,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054510 2024-11-11T16:37:34,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054510) roll requested 2024-11-11T16:37:34,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054521 2024-11-11T16:37:34,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054521 2024-11-11T16:37:34,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054521) roll requested 2024-11-11T16:37:34,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054531 2024-11-11T16:37:34,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054531 2024-11-11T16:37:34,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054531) roll requested 2024-11-11T16:37:34,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054541 2024-11-11T16:37:34,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054541 2024-11-11T16:37:34,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054541) roll requested 2024-11-11T16:37:34,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054549 2024-11-11T16:37:34,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054549 2024-11-11T16:37:34,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054549) roll requested 2024-11-11T16:37:34,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054558 2024-11-11T16:37:34,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054558 2024-11-11T16:37:34,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054558) roll requested 2024-11-11T16:37:34,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054566 2024-11-11T16:37:34,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054566 2024-11-11T16:37:34,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054566) roll requested 2024-11-11T16:37:34,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054573 2024-11-11T16:37:34,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054573 2024-11-11T16:37:34,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054573) roll requested 2024-11-11T16:37:34,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054580 2024-11-11T16:37:34,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054580 2024-11-11T16:37:34,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054580) roll requested 2024-11-11T16:37:34,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054587 2024-11-11T16:37:34,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054587 2024-11-11T16:37:34,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054587) roll requested 2024-11-11T16:37:34,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054601 2024-11-11T16:37:34,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054601 2024-11-11T16:37:34,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054601) roll requested 2024-11-11T16:37:34,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054609 2024-11-11T16:37:34,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054609 2024-11-11T16:37:34,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054609) roll requested 2024-11-11T16:37:34,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054617 2024-11-11T16:37:34,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054617 2024-11-11T16:37:34,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054617) roll requested 2024-11-11T16:37:34,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054625 2024-11-11T16:37:34,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054625 2024-11-11T16:37:34,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054625) roll requested 2024-11-11T16:37:34,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054632 2024-11-11T16:37:34,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054632 2024-11-11T16:37:34,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054632) roll requested 2024-11-11T16:37:34,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054638 2024-11-11T16:37:34,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054638 2024-11-11T16:37:34,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054638) roll requested 2024-11-11T16:37:34,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054645 2024-11-11T16:37:34,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054645 2024-11-11T16:37:34,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054645) roll requested 2024-11-11T16:37:34,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054652 2024-11-11T16:37:34,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054652 2024-11-11T16:37:34,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054652) roll requested 2024-11-11T16:37:34,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054664 2024-11-11T16:37:34,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054664 2024-11-11T16:37:34,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054664) roll requested 2024-11-11T16:37:34,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054680 2024-11-11T16:37:34,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054680 2024-11-11T16:37:34,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054680) roll requested 2024-11-11T16:37:34,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054688 2024-11-11T16:37:34,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054688 2024-11-11T16:37:34,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054688) roll requested 2024-11-11T16:37:34,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054696 2024-11-11T16:37:34,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054696 2024-11-11T16:37:34,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054696) roll requested 2024-11-11T16:37:34,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054704 2024-11-11T16:37:34,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054704 2024-11-11T16:37:34,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054704) roll requested 2024-11-11T16:37:34,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054712 2024-11-11T16:37:34,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054712 2024-11-11T16:37:34,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054712) roll requested 2024-11-11T16:37:34,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054720 2024-11-11T16:37:34,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054720 2024-11-11T16:37:34,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054720) roll requested 2024-11-11T16:37:34,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054727 2024-11-11T16:37:34,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054727 2024-11-11T16:37:34,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054727) roll requested 2024-11-11T16:37:34,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054736 2024-11-11T16:37:34,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054736 2024-11-11T16:37:34,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054736) roll requested 2024-11-11T16:37:34,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054756 2024-11-11T16:37:34,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054756 2024-11-11T16:37:34,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054756) roll requested 2024-11-11T16:37:34,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054765 2024-11-11T16:37:34,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054765 2024-11-11T16:37:34,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054765) roll requested 2024-11-11T16:37:34,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054777 2024-11-11T16:37:34,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054777 2024-11-11T16:37:34,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054777) roll requested 2024-11-11T16:37:34,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054797 2024-11-11T16:37:34,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054797 2024-11-11T16:37:34,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054797) roll requested 2024-11-11T16:37:34,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054808 2024-11-11T16:37:34,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054808 2024-11-11T16:37:34,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054808) roll requested 2024-11-11T16:37:34,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054817 2024-11-11T16:37:34,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054817 2024-11-11T16:37:34,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054817) roll requested 2024-11-11T16:37:34,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054824 2024-11-11T16:37:34,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054824 2024-11-11T16:37:34,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054824) roll requested 2024-11-11T16:37:34,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054831 2024-11-11T16:37:34,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054831 2024-11-11T16:37:34,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054831) roll requested 2024-11-11T16:37:34,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054839 2024-11-11T16:37:34,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054839 2024-11-11T16:37:34,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054839) roll requested 2024-11-11T16:37:34,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054847 2024-11-11T16:37:34,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054847 2024-11-11T16:37:34,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054847) roll requested 2024-11-11T16:37:34,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054859 2024-11-11T16:37:34,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054859 2024-11-11T16:37:34,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054859) roll requested 2024-11-11T16:37:34,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054871 2024-11-11T16:37:34,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054871 2024-11-11T16:37:34,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054871) roll requested 2024-11-11T16:37:34,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054883 2024-11-11T16:37:34,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054883 2024-11-11T16:37:34,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054883) roll requested 2024-11-11T16:37:34,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054892 2024-11-11T16:37:34,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054892 2024-11-11T16:37:34,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054892) roll requested 2024-11-11T16:37:34,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054900 2024-11-11T16:37:34,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054900 2024-11-11T16:37:34,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054900) roll requested 2024-11-11T16:37:34,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054907 2024-11-11T16:37:34,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054907 2024-11-11T16:37:34,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054907) roll requested 2024-11-11T16:37:34,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054915 2024-11-11T16:37:34,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054915 2024-11-11T16:37:34,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054915) roll requested 2024-11-11T16:37:34,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054929 2024-11-11T16:37:34,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054929 2024-11-11T16:37:34,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054929) roll requested 2024-11-11T16:37:34,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054937 2024-11-11T16:37:34,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054937 2024-11-11T16:37:34,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054937) roll requested 2024-11-11T16:37:34,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054947 2024-11-11T16:37:34,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054947 2024-11-11T16:37:34,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054947) roll requested 2024-11-11T16:37:34,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054955 2024-11-11T16:37:34,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054955 2024-11-11T16:37:34,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054955) roll requested 2024-11-11T16:37:34,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054965 2024-11-11T16:37:34,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054965 2024-11-11T16:37:34,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054965) roll requested 2024-11-11T16:37:34,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054979 2024-11-11T16:37:34,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054979 2024-11-11T16:37:34,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054979) roll requested 2024-11-11T16:37:34,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054985 2024-11-11T16:37:34,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054985 2024-11-11T16:37:34,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054985) roll requested 2024-11-11T16:37:34,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054992 2024-11-11T16:37:34,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:34,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054992 2024-11-11T16:37:34,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:34,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:34,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054992) roll requested 2024-11-11T16:37:34,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343054999 2024-11-11T16:37:35,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054999 2024-11-11T16:37:35,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343054999) roll requested 2024-11-11T16:37:35,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055005 2024-11-11T16:37:35,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343054999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055005 2024-11-11T16:37:35,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055005) roll requested 2024-11-11T16:37:35,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055012 2024-11-11T16:37:35,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055012 2024-11-11T16:37:35,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055012) roll requested 2024-11-11T16:37:35,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055019 2024-11-11T16:37:35,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055019 2024-11-11T16:37:35,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055019) roll requested 2024-11-11T16:37:35,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055026 2024-11-11T16:37:35,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055026 2024-11-11T16:37:35,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055026) roll requested 2024-11-11T16:37:35,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055033 2024-11-11T16:37:35,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055033 2024-11-11T16:37:35,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055033) roll requested 2024-11-11T16:37:35,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055040 2024-11-11T16:37:35,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055040 2024-11-11T16:37:35,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055040) roll requested 2024-11-11T16:37:35,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055046 2024-11-11T16:37:35,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055046 2024-11-11T16:37:35,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055046) roll requested 2024-11-11T16:37:35,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055053 2024-11-11T16:37:35,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055053 2024-11-11T16:37:35,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055053) roll requested 2024-11-11T16:37:35,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055060 2024-11-11T16:37:35,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055060 2024-11-11T16:37:35,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055060) roll requested 2024-11-11T16:37:35,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055066 2024-11-11T16:37:35,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055066 2024-11-11T16:37:35,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055066) roll requested 2024-11-11T16:37:35,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055076 2024-11-11T16:37:35,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055076 2024-11-11T16:37:35,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055076) roll requested 2024-11-11T16:37:35,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055083 2024-11-11T16:37:35,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055083 2024-11-11T16:37:35,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055083) roll requested 2024-11-11T16:37:35,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055090 2024-11-11T16:37:35,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055090 2024-11-11T16:37:35,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055090) roll requested 2024-11-11T16:37:35,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055101 2024-11-11T16:37:35,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055101 2024-11-11T16:37:35,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055101) roll requested 2024-11-11T16:37:35,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055111 2024-11-11T16:37:35,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055111 2024-11-11T16:37:35,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055111) roll requested 2024-11-11T16:37:35,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055121 2024-11-11T16:37:35,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055121 2024-11-11T16:37:35,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055121) roll requested 2024-11-11T16:37:35,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055130 2024-11-11T16:37:35,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055130 2024-11-11T16:37:35,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055130) roll requested 2024-11-11T16:37:35,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055143 2024-11-11T16:37:35,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055143 2024-11-11T16:37:35,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055143) roll requested 2024-11-11T16:37:35,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055153 2024-11-11T16:37:35,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055153 2024-11-11T16:37:35,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055153) roll requested 2024-11-11T16:37:35,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055161 2024-11-11T16:37:35,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055161 2024-11-11T16:37:35,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055161) roll requested 2024-11-11T16:37:35,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055169 2024-11-11T16:37:35,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055169 2024-11-11T16:37:35,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055169) roll requested 2024-11-11T16:37:35,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055177 2024-11-11T16:37:35,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055177 2024-11-11T16:37:35,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055177) roll requested 2024-11-11T16:37:35,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055184 2024-11-11T16:37:35,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055184 2024-11-11T16:37:35,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055184) roll requested 2024-11-11T16:37:35,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055192 2024-11-11T16:37:35,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055192 2024-11-11T16:37:35,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055192) roll requested 2024-11-11T16:37:35,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055199 2024-11-11T16:37:35,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055199 2024-11-11T16:37:35,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055199) roll requested 2024-11-11T16:37:35,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055206 2024-11-11T16:37:35,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055206 2024-11-11T16:37:35,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055206) roll requested 2024-11-11T16:37:35,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055213 2024-11-11T16:37:35,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055213 2024-11-11T16:37:35,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055213) roll requested 2024-11-11T16:37:35,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055219 2024-11-11T16:37:35,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055219 2024-11-11T16:37:35,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055219) roll requested 2024-11-11T16:37:35,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055227 2024-11-11T16:37:35,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055227 2024-11-11T16:37:35,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055227) roll requested 2024-11-11T16:37:35,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055234 2024-11-11T16:37:35,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055234 2024-11-11T16:37:35,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055234) roll requested 2024-11-11T16:37:35,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055243 2024-11-11T16:37:35,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055243 2024-11-11T16:37:35,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055243) roll requested 2024-11-11T16:37:35,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055250 2024-11-11T16:37:35,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055250 2024-11-11T16:37:35,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055250) roll requested 2024-11-11T16:37:35,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055257 2024-11-11T16:37:35,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055257 2024-11-11T16:37:35,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055257) roll requested 2024-11-11T16:37:35,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055264 2024-11-11T16:37:35,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055264 2024-11-11T16:37:35,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055264) roll requested 2024-11-11T16:37:35,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055276 2024-11-11T16:37:35,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055276 2024-11-11T16:37:35,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055276) roll requested 2024-11-11T16:37:35,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055286 2024-11-11T16:37:35,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055286 2024-11-11T16:37:35,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055286) roll requested 2024-11-11T16:37:35,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055293 2024-11-11T16:37:35,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055293 2024-11-11T16:37:35,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055293) roll requested 2024-11-11T16:37:35,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055299 2024-11-11T16:37:35,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055299 2024-11-11T16:37:35,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055299) roll requested 2024-11-11T16:37:35,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055306 2024-11-11T16:37:35,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055306 2024-11-11T16:37:35,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055306) roll requested 2024-11-11T16:37:35,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055317 2024-11-11T16:37:35,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055317 2024-11-11T16:37:35,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055317) roll requested 2024-11-11T16:37:35,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055324 2024-11-11T16:37:35,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055324 2024-11-11T16:37:35,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055324) roll requested 2024-11-11T16:37:35,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055331 2024-11-11T16:37:35,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055331 2024-11-11T16:37:35,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055331) roll requested 2024-11-11T16:37:35,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055338 2024-11-11T16:37:35,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055338 2024-11-11T16:37:35,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055338) roll requested 2024-11-11T16:37:35,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055350 2024-11-11T16:37:35,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055350 2024-11-11T16:37:35,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055350) roll requested 2024-11-11T16:37:35,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055361 2024-11-11T16:37:35,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055361 2024-11-11T16:37:35,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055361) roll requested 2024-11-11T16:37:35,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055376 2024-11-11T16:37:35,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055376 2024-11-11T16:37:35,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055376) roll requested 2024-11-11T16:37:35,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055386 2024-11-11T16:37:35,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055386 2024-11-11T16:37:35,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055386) roll requested 2024-11-11T16:37:35,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055394 2024-11-11T16:37:35,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055394 2024-11-11T16:37:35,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055394) roll requested 2024-11-11T16:37:35,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055403 2024-11-11T16:37:35,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055403 2024-11-11T16:37:35,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055403) roll requested 2024-11-11T16:37:35,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055411 2024-11-11T16:37:35,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055411 2024-11-11T16:37:35,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055411) roll requested 2024-11-11T16:37:35,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055418 2024-11-11T16:37:35,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055418 2024-11-11T16:37:35,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055418) roll requested 2024-11-11T16:37:35,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055426 2024-11-11T16:37:35,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055426 2024-11-11T16:37:35,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055426) roll requested 2024-11-11T16:37:35,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055432 2024-11-11T16:37:35,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055432 2024-11-11T16:37:35,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055432) roll requested 2024-11-11T16:37:35,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055439 2024-11-11T16:37:35,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055439 2024-11-11T16:37:35,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055439) roll requested 2024-11-11T16:37:35,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055447 2024-11-11T16:37:35,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055447 2024-11-11T16:37:35,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055447) roll requested 2024-11-11T16:37:35,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055455 2024-11-11T16:37:35,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055455 2024-11-11T16:37:35,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055455) roll requested 2024-11-11T16:37:35,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055463 2024-11-11T16:37:35,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055463 2024-11-11T16:37:35,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055463) roll requested 2024-11-11T16:37:35,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055470 2024-11-11T16:37:35,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055470 2024-11-11T16:37:35,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055470) roll requested 2024-11-11T16:37:35,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055477 2024-11-11T16:37:35,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055477 2024-11-11T16:37:35,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055477) roll requested 2024-11-11T16:37:35,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055484 2024-11-11T16:37:35,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055484 2024-11-11T16:37:35,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055484) roll requested 2024-11-11T16:37:35,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055491 2024-11-11T16:37:35,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055491 2024-11-11T16:37:35,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055491) roll requested 2024-11-11T16:37:35,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055497 2024-11-11T16:37:35,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055497 2024-11-11T16:37:35,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055497) roll requested 2024-11-11T16:37:35,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055504 2024-11-11T16:37:35,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055504 2024-11-11T16:37:35,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055504) roll requested 2024-11-11T16:37:35,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055511 2024-11-11T16:37:35,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055511 2024-11-11T16:37:35,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055511) roll requested 2024-11-11T16:37:35,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055517 2024-11-11T16:37:35,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055517 2024-11-11T16:37:35,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055517) roll requested 2024-11-11T16:37:35,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055524 2024-11-11T16:37:35,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055524 2024-11-11T16:37:35,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055524) roll requested 2024-11-11T16:37:35,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055530 2024-11-11T16:37:35,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055530 2024-11-11T16:37:35,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055530) roll requested 2024-11-11T16:37:35,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055537 2024-11-11T16:37:35,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055537 2024-11-11T16:37:35,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055537) roll requested 2024-11-11T16:37:35,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055544 2024-11-11T16:37:35,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055544 2024-11-11T16:37:35,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055544) roll requested 2024-11-11T16:37:35,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055551 2024-11-11T16:37:35,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055551 2024-11-11T16:37:35,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055551) roll requested 2024-11-11T16:37:35,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055557 2024-11-11T16:37:35,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055557 2024-11-11T16:37:35,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055557) roll requested 2024-11-11T16:37:35,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055565 2024-11-11T16:37:35,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055565 2024-11-11T16:37:35,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055565) roll requested 2024-11-11T16:37:35,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055572 2024-11-11T16:37:35,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055572 2024-11-11T16:37:35,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055572) roll requested 2024-11-11T16:37:35,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055578 2024-11-11T16:37:35,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055578 2024-11-11T16:37:35,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055578) roll requested 2024-11-11T16:37:35,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055588 2024-11-11T16:37:35,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055588 2024-11-11T16:37:35,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055588) roll requested 2024-11-11T16:37:35,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055594 2024-11-11T16:37:35,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055594 2024-11-11T16:37:35,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055594) roll requested 2024-11-11T16:37:35,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055602 2024-11-11T16:37:35,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055602 2024-11-11T16:37:35,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055602) roll requested 2024-11-11T16:37:35,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055609 2024-11-11T16:37:35,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055609 2024-11-11T16:37:35,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055609) roll requested 2024-11-11T16:37:35,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055616 2024-11-11T16:37:35,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055616 2024-11-11T16:37:35,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055616) roll requested 2024-11-11T16:37:35,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055622 2024-11-11T16:37:35,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055622 2024-11-11T16:37:35,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055622) roll requested 2024-11-11T16:37:35,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055629 2024-11-11T16:37:35,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055629 2024-11-11T16:37:35,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055629) roll requested 2024-11-11T16:37:35,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055638 2024-11-11T16:37:35,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055638 2024-11-11T16:37:35,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055638) roll requested 2024-11-11T16:37:35,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055653 2024-11-11T16:37:35,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055653 2024-11-11T16:37:35,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055653) roll requested 2024-11-11T16:37:35,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055677 2024-11-11T16:37:35,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055677 2024-11-11T16:37:35,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055677) roll requested 2024-11-11T16:37:35,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055687 2024-11-11T16:37:35,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055687 2024-11-11T16:37:35,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055687) roll requested 2024-11-11T16:37:35,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055694 2024-11-11T16:37:35,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055694 2024-11-11T16:37:35,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055694) roll requested 2024-11-11T16:37:35,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055701 2024-11-11T16:37:35,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055701 2024-11-11T16:37:35,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055701) roll requested 2024-11-11T16:37:35,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055711 2024-11-11T16:37:35,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055711 2024-11-11T16:37:35,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055711) roll requested 2024-11-11T16:37:35,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055718 2024-11-11T16:37:35,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055718 2024-11-11T16:37:35,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055718) roll requested 2024-11-11T16:37:35,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055726 2024-11-11T16:37:35,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055726 2024-11-11T16:37:35,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055726) roll requested 2024-11-11T16:37:35,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055733 2024-11-11T16:37:35,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055733 2024-11-11T16:37:35,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055733) roll requested 2024-11-11T16:37:35,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055740 2024-11-11T16:37:35,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055740 2024-11-11T16:37:35,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055740) roll requested 2024-11-11T16:37:35,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055747 2024-11-11T16:37:35,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055747 2024-11-11T16:37:35,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055747) roll requested 2024-11-11T16:37:35,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055754 2024-11-11T16:37:35,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055754 2024-11-11T16:37:35,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055754) roll requested 2024-11-11T16:37:35,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055761 2024-11-11T16:37:35,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055761 2024-11-11T16:37:35,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055761) roll requested 2024-11-11T16:37:35,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055768 2024-11-11T16:37:35,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055768 2024-11-11T16:37:35,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055768) roll requested 2024-11-11T16:37:35,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055774 2024-11-11T16:37:35,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055774 2024-11-11T16:37:35,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055774) roll requested 2024-11-11T16:37:35,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055782 2024-11-11T16:37:35,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055782 2024-11-11T16:37:35,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055782) roll requested 2024-11-11T16:37:35,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055789 2024-11-11T16:37:35,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055789 2024-11-11T16:37:35,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055789) roll requested 2024-11-11T16:37:35,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055796 2024-11-11T16:37:35,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055796 2024-11-11T16:37:35,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055796) roll requested 2024-11-11T16:37:35,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055802 2024-11-11T16:37:35,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055802 2024-11-11T16:37:35,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055802) roll requested 2024-11-11T16:37:35,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055809 2024-11-11T16:37:35,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055809 2024-11-11T16:37:35,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055809) roll requested 2024-11-11T16:37:35,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055817 2024-11-11T16:37:35,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055817 2024-11-11T16:37:35,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055817) roll requested 2024-11-11T16:37:35,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055826 2024-11-11T16:37:35,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055826 2024-11-11T16:37:35,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055826) roll requested 2024-11-11T16:37:35,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055837 2024-11-11T16:37:35,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055837 2024-11-11T16:37:35,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055837) roll requested 2024-11-11T16:37:35,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055843 2024-11-11T16:37:35,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055843 2024-11-11T16:37:35,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055843) roll requested 2024-11-11T16:37:35,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055850 2024-11-11T16:37:35,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055850 2024-11-11T16:37:35,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055850) roll requested 2024-11-11T16:37:35,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055857 2024-11-11T16:37:35,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055857 2024-11-11T16:37:35,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055857) roll requested 2024-11-11T16:37:35,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055863 2024-11-11T16:37:35,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055863 2024-11-11T16:37:35,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055863) roll requested 2024-11-11T16:37:35,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055870 2024-11-11T16:37:35,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055870 2024-11-11T16:37:35,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055870) roll requested 2024-11-11T16:37:35,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055878 2024-11-11T16:37:35,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055878 2024-11-11T16:37:35,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055878) roll requested 2024-11-11T16:37:35,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055885 2024-11-11T16:37:35,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055885 2024-11-11T16:37:35,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055885) roll requested 2024-11-11T16:37:35,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055892 2024-11-11T16:37:35,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055892 2024-11-11T16:37:35,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055892) roll requested 2024-11-11T16:37:35,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055898 2024-11-11T16:37:35,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055898 2024-11-11T16:37:35,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055898) roll requested 2024-11-11T16:37:35,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055905 2024-11-11T16:37:35,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055905 2024-11-11T16:37:35,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055905) roll requested 2024-11-11T16:37:35,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055912 2024-11-11T16:37:35,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055912 2024-11-11T16:37:35,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055912) roll requested 2024-11-11T16:37:35,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055925 2024-11-11T16:37:35,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055925 2024-11-11T16:37:35,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055925) roll requested 2024-11-11T16:37:35,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055935 2024-11-11T16:37:35,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055935 2024-11-11T16:37:35,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055935) roll requested 2024-11-11T16:37:35,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055942 2024-11-11T16:37:35,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055942 2024-11-11T16:37:35,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055942) roll requested 2024-11-11T16:37:35,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055953 2024-11-11T16:37:35,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055953 2024-11-11T16:37:35,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055953) roll requested 2024-11-11T16:37:35,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055960 2024-11-11T16:37:35,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055960 2024-11-11T16:37:35,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055960) roll requested 2024-11-11T16:37:35,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055967 2024-11-11T16:37:35,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055967 2024-11-11T16:37:35,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055967) roll requested 2024-11-11T16:37:35,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055974 2024-11-11T16:37:35,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055974 2024-11-11T16:37:35,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055974) roll requested 2024-11-11T16:37:35,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055982 2024-11-11T16:37:35,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055982 2024-11-11T16:37:35,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:35,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:35,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055982) roll requested 2024-11-11T16:37:35,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343055993 2024-11-11T16:37:35,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:35,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055993 2024-11-11T16:37:36,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343055993) roll requested 2024-11-11T16:37:36,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056000 2024-11-11T16:37:36,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343055993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056000 2024-11-11T16:37:36,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056000) roll requested 2024-11-11T16:37:36,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056010 2024-11-11T16:37:36,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056010 2024-11-11T16:37:36,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056010) roll requested 2024-11-11T16:37:36,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056019 2024-11-11T16:37:36,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056019 2024-11-11T16:37:36,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056019) roll requested 2024-11-11T16:37:36,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056025 2024-11-11T16:37:36,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056025 2024-11-11T16:37:36,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056025) roll requested 2024-11-11T16:37:36,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056033 2024-11-11T16:37:36,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056033 2024-11-11T16:37:36,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056033) roll requested 2024-11-11T16:37:36,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056040 2024-11-11T16:37:36,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056040 2024-11-11T16:37:36,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056040) roll requested 2024-11-11T16:37:36,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056047 2024-11-11T16:37:36,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056047 2024-11-11T16:37:36,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056047) roll requested 2024-11-11T16:37:36,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056054 2024-11-11T16:37:36,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056054 2024-11-11T16:37:36,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056054) roll requested 2024-11-11T16:37:36,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056062 2024-11-11T16:37:36,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056062 2024-11-11T16:37:36,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056062) roll requested 2024-11-11T16:37:36,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056069 2024-11-11T16:37:36,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056069 2024-11-11T16:37:36,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056069) roll requested 2024-11-11T16:37:36,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056076 2024-11-11T16:37:36,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056076 2024-11-11T16:37:36,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056076) roll requested 2024-11-11T16:37:36,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056084 2024-11-11T16:37:36,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056084 2024-11-11T16:37:36,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056084) roll requested 2024-11-11T16:37:36,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056091 2024-11-11T16:37:36,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056091 2024-11-11T16:37:36,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056091) roll requested 2024-11-11T16:37:36,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056101 2024-11-11T16:37:36,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056101 2024-11-11T16:37:36,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056101) roll requested 2024-11-11T16:37:36,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056108 2024-11-11T16:37:36,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056108 2024-11-11T16:37:36,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056108) roll requested 2024-11-11T16:37:36,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056116 2024-11-11T16:37:36,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056116 2024-11-11T16:37:36,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056116) roll requested 2024-11-11T16:37:36,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056123 2024-11-11T16:37:36,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056123 2024-11-11T16:37:36,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056123) roll requested 2024-11-11T16:37:36,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056130 2024-11-11T16:37:36,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056130 2024-11-11T16:37:36,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056130) roll requested 2024-11-11T16:37:36,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056138 2024-11-11T16:37:36,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056138 2024-11-11T16:37:36,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056138) roll requested 2024-11-11T16:37:36,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056145 2024-11-11T16:37:36,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056145 2024-11-11T16:37:36,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056145) roll requested 2024-11-11T16:37:36,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056153 2024-11-11T16:37:36,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056153 2024-11-11T16:37:36,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056153) roll requested 2024-11-11T16:37:36,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056160 2024-11-11T16:37:36,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056160 2024-11-11T16:37:36,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056160) roll requested 2024-11-11T16:37:36,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056167 2024-11-11T16:37:36,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056167 2024-11-11T16:37:36,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056167) roll requested 2024-11-11T16:37:36,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056174 2024-11-11T16:37:36,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056174 2024-11-11T16:37:36,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056174) roll requested 2024-11-11T16:37:36,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056181 2024-11-11T16:37:36,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056181 2024-11-11T16:37:36,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056181) roll requested 2024-11-11T16:37:36,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056188 2024-11-11T16:37:36,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056188 2024-11-11T16:37:36,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056188) roll requested 2024-11-11T16:37:36,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056194 2024-11-11T16:37:36,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056194 2024-11-11T16:37:36,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056194) roll requested 2024-11-11T16:37:36,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056205 2024-11-11T16:37:36,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056205 2024-11-11T16:37:36,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056205) roll requested 2024-11-11T16:37:36,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056212 2024-11-11T16:37:36,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056212 2024-11-11T16:37:36,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056212) roll requested 2024-11-11T16:37:36,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056221 2024-11-11T16:37:36,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056221 2024-11-11T16:37:36,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056221) roll requested 2024-11-11T16:37:36,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056229 2024-11-11T16:37:36,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056229 2024-11-11T16:37:36,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056229) roll requested 2024-11-11T16:37:36,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056237 2024-11-11T16:37:36,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056237 2024-11-11T16:37:36,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056237) roll requested 2024-11-11T16:37:36,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056246 2024-11-11T16:37:36,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056246 2024-11-11T16:37:36,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056246) roll requested 2024-11-11T16:37:36,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056253 2024-11-11T16:37:36,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056253 2024-11-11T16:37:36,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056253) roll requested 2024-11-11T16:37:36,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056260 2024-11-11T16:37:36,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056260 2024-11-11T16:37:36,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056260) roll requested 2024-11-11T16:37:36,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056267 2024-11-11T16:37:36,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056267 2024-11-11T16:37:36,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056267) roll requested 2024-11-11T16:37:36,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056274 2024-11-11T16:37:36,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056274 2024-11-11T16:37:36,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056274) roll requested 2024-11-11T16:37:36,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056281 2024-11-11T16:37:36,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056281 2024-11-11T16:37:36,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056281) roll requested 2024-11-11T16:37:36,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056293 2024-11-11T16:37:36,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056293 2024-11-11T16:37:36,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056293) roll requested 2024-11-11T16:37:36,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056300 2024-11-11T16:37:36,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056300 2024-11-11T16:37:36,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056300) roll requested 2024-11-11T16:37:36,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056307 2024-11-11T16:37:36,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056307 2024-11-11T16:37:36,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056307) roll requested 2024-11-11T16:37:36,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056315 2024-11-11T16:37:36,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056315 2024-11-11T16:37:36,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056315) roll requested 2024-11-11T16:37:36,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056321 2024-11-11T16:37:36,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056321 2024-11-11T16:37:36,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056321) roll requested 2024-11-11T16:37:36,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056328 2024-11-11T16:37:36,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056328 2024-11-11T16:37:36,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056328) roll requested 2024-11-11T16:37:36,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056335 2024-11-11T16:37:36,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056335 2024-11-11T16:37:36,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056335) roll requested 2024-11-11T16:37:36,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056342 2024-11-11T16:37:36,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056342 2024-11-11T16:37:36,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056342) roll requested 2024-11-11T16:37:36,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056348 2024-11-11T16:37:36,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056348 2024-11-11T16:37:36,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056348) roll requested 2024-11-11T16:37:36,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056355 2024-11-11T16:37:36,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056355 2024-11-11T16:37:36,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056355) roll requested 2024-11-11T16:37:36,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056363 2024-11-11T16:37:36,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056363 2024-11-11T16:37:36,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056363) roll requested 2024-11-11T16:37:36,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056370 2024-11-11T16:37:36,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056370 2024-11-11T16:37:36,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056370) roll requested 2024-11-11T16:37:36,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056377 2024-11-11T16:37:36,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056377 2024-11-11T16:37:36,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056377) roll requested 2024-11-11T16:37:36,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056385 2024-11-11T16:37:36,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056385 2024-11-11T16:37:36,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056385) roll requested 2024-11-11T16:37:36,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056393 2024-11-11T16:37:36,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056393 2024-11-11T16:37:36,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056393) roll requested 2024-11-11T16:37:36,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056399 2024-11-11T16:37:36,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056399 2024-11-11T16:37:36,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056399) roll requested 2024-11-11T16:37:36,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056406 2024-11-11T16:37:36,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056406 2024-11-11T16:37:36,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056406) roll requested 2024-11-11T16:37:36,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056412 2024-11-11T16:37:36,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056412 2024-11-11T16:37:36,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056412) roll requested 2024-11-11T16:37:36,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056419 2024-11-11T16:37:36,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056419 2024-11-11T16:37:36,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056419) roll requested 2024-11-11T16:37:36,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056426 2024-11-11T16:37:36,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056426 2024-11-11T16:37:36,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056426) roll requested 2024-11-11T16:37:36,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056432 2024-11-11T16:37:36,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056432 2024-11-11T16:37:36,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056432) roll requested 2024-11-11T16:37:36,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056439 2024-11-11T16:37:36,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056439 2024-11-11T16:37:36,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056439) roll requested 2024-11-11T16:37:36,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056446 2024-11-11T16:37:36,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056446 2024-11-11T16:37:36,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056446) roll requested 2024-11-11T16:37:36,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056454 2024-11-11T16:37:36,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056454 2024-11-11T16:37:36,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056454) roll requested 2024-11-11T16:37:36,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056463 2024-11-11T16:37:36,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056463 2024-11-11T16:37:36,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056463) roll requested 2024-11-11T16:37:36,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056470 2024-11-11T16:37:36,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056470 2024-11-11T16:37:36,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056470) roll requested 2024-11-11T16:37:36,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056477 2024-11-11T16:37:36,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056477 2024-11-11T16:37:36,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056477) roll requested 2024-11-11T16:37:36,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056484 2024-11-11T16:37:36,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056484 2024-11-11T16:37:36,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056484) roll requested 2024-11-11T16:37:36,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056491 2024-11-11T16:37:36,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056491 2024-11-11T16:37:36,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056491) roll requested 2024-11-11T16:37:36,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056498 2024-11-11T16:37:36,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056498 2024-11-11T16:37:36,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056498) roll requested 2024-11-11T16:37:36,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056505 2024-11-11T16:37:36,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056505 2024-11-11T16:37:36,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056505) roll requested 2024-11-11T16:37:36,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056512 2024-11-11T16:37:36,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056512 2024-11-11T16:37:36,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056512) roll requested 2024-11-11T16:37:36,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056518 2024-11-11T16:37:36,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056518 2024-11-11T16:37:36,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056518) roll requested 2024-11-11T16:37:36,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056525 2024-11-11T16:37:36,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056525 2024-11-11T16:37:36,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056525) roll requested 2024-11-11T16:37:36,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056532 2024-11-11T16:37:36,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056532 2024-11-11T16:37:36,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056532) roll requested 2024-11-11T16:37:36,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056539 2024-11-11T16:37:36,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056539 2024-11-11T16:37:36,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056539) roll requested 2024-11-11T16:37:36,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056545 2024-11-11T16:37:36,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056545 2024-11-11T16:37:36,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056545) roll requested 2024-11-11T16:37:36,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056552 2024-11-11T16:37:36,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056552 2024-11-11T16:37:36,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056552) roll requested 2024-11-11T16:37:36,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056559 2024-11-11T16:37:36,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056559 2024-11-11T16:37:36,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056559) roll requested 2024-11-11T16:37:36,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056565 2024-11-11T16:37:36,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056565 2024-11-11T16:37:36,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056565) roll requested 2024-11-11T16:37:36,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056572 2024-11-11T16:37:36,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056572 2024-11-11T16:37:36,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056572) roll requested 2024-11-11T16:37:36,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056578 2024-11-11T16:37:36,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056578 2024-11-11T16:37:36,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056578) roll requested 2024-11-11T16:37:36,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056585 2024-11-11T16:37:36,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056585 2024-11-11T16:37:36,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056585) roll requested 2024-11-11T16:37:36,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056591 2024-11-11T16:37:36,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056591 2024-11-11T16:37:36,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056591) roll requested 2024-11-11T16:37:36,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056599 2024-11-11T16:37:36,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056599 2024-11-11T16:37:36,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056599) roll requested 2024-11-11T16:37:36,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056607 2024-11-11T16:37:36,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056607 2024-11-11T16:37:36,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056607) roll requested 2024-11-11T16:37:36,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056616 2024-11-11T16:37:36,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056616 2024-11-11T16:37:36,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056616) roll requested 2024-11-11T16:37:36,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056623 2024-11-11T16:37:36,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056623 2024-11-11T16:37:36,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056623) roll requested 2024-11-11T16:37:36,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056632 2024-11-11T16:37:36,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056632 2024-11-11T16:37:36,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056632) roll requested 2024-11-11T16:37:36,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056638 2024-11-11T16:37:36,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056638 2024-11-11T16:37:36,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056638) roll requested 2024-11-11T16:37:36,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056656 2024-11-11T16:37:36,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056656 2024-11-11T16:37:36,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056656) roll requested 2024-11-11T16:37:36,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056663 2024-11-11T16:37:36,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056663 2024-11-11T16:37:36,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056663) roll requested 2024-11-11T16:37:36,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056670 2024-11-11T16:37:36,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056670 2024-11-11T16:37:36,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056670) roll requested 2024-11-11T16:37:36,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056677 2024-11-11T16:37:36,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056677 2024-11-11T16:37:36,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056677) roll requested 2024-11-11T16:37:36,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056683 2024-11-11T16:37:36,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056683 2024-11-11T16:37:36,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056683) roll requested 2024-11-11T16:37:36,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056691 2024-11-11T16:37:36,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056691 2024-11-11T16:37:36,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056691) roll requested 2024-11-11T16:37:36,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056697 2024-11-11T16:37:36,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056697 2024-11-11T16:37:36,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056697) roll requested 2024-11-11T16:37:36,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056704 2024-11-11T16:37:36,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056704 2024-11-11T16:37:36,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056704) roll requested 2024-11-11T16:37:36,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056710 2024-11-11T16:37:36,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056710 2024-11-11T16:37:36,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056710) roll requested 2024-11-11T16:37:36,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056717 2024-11-11T16:37:36,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056717 2024-11-11T16:37:36,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056717) roll requested 2024-11-11T16:37:36,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056723 2024-11-11T16:37:36,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056723 2024-11-11T16:37:36,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056723) roll requested 2024-11-11T16:37:36,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056729 2024-11-11T16:37:36,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056729 2024-11-11T16:37:36,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056729) roll requested 2024-11-11T16:37:36,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056735 2024-11-11T16:37:36,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056735 2024-11-11T16:37:36,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056735) roll requested 2024-11-11T16:37:36,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056743 2024-11-11T16:37:36,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056743 2024-11-11T16:37:36,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056743) roll requested 2024-11-11T16:37:36,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056750 2024-11-11T16:37:36,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056750 2024-11-11T16:37:36,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056750) roll requested 2024-11-11T16:37:36,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056756 2024-11-11T16:37:36,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056756 2024-11-11T16:37:36,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056756) roll requested 2024-11-11T16:37:36,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056763 2024-11-11T16:37:36,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056763 2024-11-11T16:37:36,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056763) roll requested 2024-11-11T16:37:36,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056769 2024-11-11T16:37:36,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056769 2024-11-11T16:37:36,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056769) roll requested 2024-11-11T16:37:36,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056776 2024-11-11T16:37:36,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056776 2024-11-11T16:37:36,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056776) roll requested 2024-11-11T16:37:36,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056782 2024-11-11T16:37:36,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056782 2024-11-11T16:37:36,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056782) roll requested 2024-11-11T16:37:36,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056788 2024-11-11T16:37:36,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056788 2024-11-11T16:37:36,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056788) roll requested 2024-11-11T16:37:36,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056795 2024-11-11T16:37:36,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056795 2024-11-11T16:37:36,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056795) roll requested 2024-11-11T16:37:36,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056802 2024-11-11T16:37:36,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056802 2024-11-11T16:37:36,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056802) roll requested 2024-11-11T16:37:36,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056811 2024-11-11T16:37:36,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056811 2024-11-11T16:37:36,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056811) roll requested 2024-11-11T16:37:36,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056819 2024-11-11T16:37:36,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056819 2024-11-11T16:37:36,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056819) roll requested 2024-11-11T16:37:36,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056825 2024-11-11T16:37:36,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056825 2024-11-11T16:37:36,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056825) roll requested 2024-11-11T16:37:36,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056832 2024-11-11T16:37:36,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056832 2024-11-11T16:37:36,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056832) roll requested 2024-11-11T16:37:36,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056839 2024-11-11T16:37:36,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056839 2024-11-11T16:37:36,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056839) roll requested 2024-11-11T16:37:36,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056846 2024-11-11T16:37:36,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056846 2024-11-11T16:37:36,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056846) roll requested 2024-11-11T16:37:36,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056853 2024-11-11T16:37:36,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056853 2024-11-11T16:37:36,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056853) roll requested 2024-11-11T16:37:36,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056860 2024-11-11T16:37:36,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056860 2024-11-11T16:37:36,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056860) roll requested 2024-11-11T16:37:36,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056867 2024-11-11T16:37:36,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056867 2024-11-11T16:37:36,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056867) roll requested 2024-11-11T16:37:36,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056877 2024-11-11T16:37:36,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056877 2024-11-11T16:37:36,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056877) roll requested 2024-11-11T16:37:36,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056897 2024-11-11T16:37:36,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056897 2024-11-11T16:37:36,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056897) roll requested 2024-11-11T16:37:36,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056910 2024-11-11T16:37:36,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056910 2024-11-11T16:37:36,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056910) roll requested 2024-11-11T16:37:36,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056921 2024-11-11T16:37:36,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056921 2024-11-11T16:37:36,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056921) roll requested 2024-11-11T16:37:36,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056933 2024-11-11T16:37:36,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056933 2024-11-11T16:37:36,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056933) roll requested 2024-11-11T16:37:36,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056954 2024-11-11T16:37:36,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056954 2024-11-11T16:37:36,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056954) roll requested 2024-11-11T16:37:36,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056965 2024-11-11T16:37:36,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056965 2024-11-11T16:37:36,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056965) roll requested 2024-11-11T16:37:36,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056981 2024-11-11T16:37:36,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:36,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056981 2024-11-11T16:37:36,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:36,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:36,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056981) roll requested 2024-11-11T16:37:36,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343056991 2024-11-11T16:37:37,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056991 2024-11-11T16:37:37,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343056991) roll requested 2024-11-11T16:37:37,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057004 2024-11-11T16:37:37,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343056991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057004 2024-11-11T16:37:37,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057004) roll requested 2024-11-11T16:37:37,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057012 2024-11-11T16:37:37,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057012 2024-11-11T16:37:37,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057012) roll requested 2024-11-11T16:37:37,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057025 2024-11-11T16:37:37,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057025 2024-11-11T16:37:37,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057025) roll requested 2024-11-11T16:37:37,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057033 2024-11-11T16:37:37,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057033 2024-11-11T16:37:37,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057033) roll requested 2024-11-11T16:37:37,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057040 2024-11-11T16:37:37,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057040 2024-11-11T16:37:37,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057040) roll requested 2024-11-11T16:37:37,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057048 2024-11-11T16:37:37,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057048 2024-11-11T16:37:37,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057048) roll requested 2024-11-11T16:37:37,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057056 2024-11-11T16:37:37,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057056 2024-11-11T16:37:37,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057056) roll requested 2024-11-11T16:37:37,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057064 2024-11-11T16:37:37,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057064 2024-11-11T16:37:37,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057064) roll requested 2024-11-11T16:37:37,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057073 2024-11-11T16:37:37,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057073 2024-11-11T16:37:37,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057073) roll requested 2024-11-11T16:37:37,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057080 2024-11-11T16:37:37,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057080 2024-11-11T16:37:37,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057080) roll requested 2024-11-11T16:37:37,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057087 2024-11-11T16:37:37,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057087 2024-11-11T16:37:37,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057087) roll requested 2024-11-11T16:37:37,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057095 2024-11-11T16:37:37,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057095 2024-11-11T16:37:37,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057095) roll requested 2024-11-11T16:37:37,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057103 2024-11-11T16:37:37,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057103 2024-11-11T16:37:37,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057103) roll requested 2024-11-11T16:37:37,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057110 2024-11-11T16:37:37,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057110 2024-11-11T16:37:37,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057110) roll requested 2024-11-11T16:37:37,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057120 2024-11-11T16:37:37,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057120 2024-11-11T16:37:37,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057120) roll requested 2024-11-11T16:37:37,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057128 2024-11-11T16:37:37,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057128 2024-11-11T16:37:37,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057128) roll requested 2024-11-11T16:37:37,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057135 2024-11-11T16:37:37,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057135 2024-11-11T16:37:37,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057135) roll requested 2024-11-11T16:37:37,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057143 2024-11-11T16:37:37,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057143 2024-11-11T16:37:37,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057143) roll requested 2024-11-11T16:37:37,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057150 2024-11-11T16:37:37,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057150 2024-11-11T16:37:37,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057150) roll requested 2024-11-11T16:37:37,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057157 2024-11-11T16:37:37,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057157 2024-11-11T16:37:37,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057157) roll requested 2024-11-11T16:37:37,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057164 2024-11-11T16:37:37,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057164 2024-11-11T16:37:37,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057164) roll requested 2024-11-11T16:37:37,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057174 2024-11-11T16:37:37,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057174 2024-11-11T16:37:37,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057174) roll requested 2024-11-11T16:37:37,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057181 2024-11-11T16:37:37,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057181 2024-11-11T16:37:37,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057181) roll requested 2024-11-11T16:37:37,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057189 2024-11-11T16:37:37,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057189 2024-11-11T16:37:37,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057189) roll requested 2024-11-11T16:37:37,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057196 2024-11-11T16:37:37,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057196 2024-11-11T16:37:37,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057196) roll requested 2024-11-11T16:37:37,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057204 2024-11-11T16:37:37,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057204 2024-11-11T16:37:37,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057204) roll requested 2024-11-11T16:37:37,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057211 2024-11-11T16:37:37,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057211 2024-11-11T16:37:37,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057211) roll requested 2024-11-11T16:37:37,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057219 2024-11-11T16:37:37,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057219 2024-11-11T16:37:37,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057219) roll requested 2024-11-11T16:37:37,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057226 2024-11-11T16:37:37,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057226 2024-11-11T16:37:37,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057226) roll requested 2024-11-11T16:37:37,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057233 2024-11-11T16:37:37,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057233 2024-11-11T16:37:37,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057233) roll requested 2024-11-11T16:37:37,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057240 2024-11-11T16:37:37,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057240 2024-11-11T16:37:37,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057240) roll requested 2024-11-11T16:37:37,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057246 2024-11-11T16:37:37,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057246 2024-11-11T16:37:37,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057246) roll requested 2024-11-11T16:37:37,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057254 2024-11-11T16:37:37,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057254 2024-11-11T16:37:37,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057254) roll requested 2024-11-11T16:37:37,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057261 2024-11-11T16:37:37,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057261 2024-11-11T16:37:37,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057261) roll requested 2024-11-11T16:37:37,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057267 2024-11-11T16:37:37,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057267 2024-11-11T16:37:37,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057267) roll requested 2024-11-11T16:37:37,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057277 2024-11-11T16:37:37,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057277 2024-11-11T16:37:37,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057277) roll requested 2024-11-11T16:37:37,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057285 2024-11-11T16:37:37,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057285 2024-11-11T16:37:37,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057285) roll requested 2024-11-11T16:37:37,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057302 2024-11-11T16:37:37,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057302 2024-11-11T16:37:37,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057302) roll requested 2024-11-11T16:37:37,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057310 2024-11-11T16:37:37,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057310 2024-11-11T16:37:37,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057310) roll requested 2024-11-11T16:37:37,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057318 2024-11-11T16:37:37,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057318 2024-11-11T16:37:37,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057318) roll requested 2024-11-11T16:37:37,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057327 2024-11-11T16:37:37,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057327 2024-11-11T16:37:37,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057327) roll requested 2024-11-11T16:37:37,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057334 2024-11-11T16:37:37,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057334 2024-11-11T16:37:37,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057334) roll requested 2024-11-11T16:37:37,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057342 2024-11-11T16:37:37,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057342 2024-11-11T16:37:37,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057342) roll requested 2024-11-11T16:37:37,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057349 2024-11-11T16:37:37,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057349 2024-11-11T16:37:37,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057349) roll requested 2024-11-11T16:37:37,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057357 2024-11-11T16:37:37,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057357 2024-11-11T16:37:37,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057357) roll requested 2024-11-11T16:37:37,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057364 2024-11-11T16:37:37,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057364 2024-11-11T16:37:37,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057364) roll requested 2024-11-11T16:37:37,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057372 2024-11-11T16:37:37,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057372 2024-11-11T16:37:37,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057372) roll requested 2024-11-11T16:37:37,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057379 2024-11-11T16:37:37,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057379 2024-11-11T16:37:37,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057379) roll requested 2024-11-11T16:37:37,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057385 2024-11-11T16:37:37,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057385 2024-11-11T16:37:37,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057385) roll requested 2024-11-11T16:37:37,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057393 2024-11-11T16:37:37,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057393 2024-11-11T16:37:37,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057393) roll requested 2024-11-11T16:37:37,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057400 2024-11-11T16:37:37,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057400 2024-11-11T16:37:37,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057400) roll requested 2024-11-11T16:37:37,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057407 2024-11-11T16:37:37,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057407 2024-11-11T16:37:37,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057407) roll requested 2024-11-11T16:37:37,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057413 2024-11-11T16:37:37,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057413 2024-11-11T16:37:37,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057413) roll requested 2024-11-11T16:37:37,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057420 2024-11-11T16:37:37,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057420 2024-11-11T16:37:37,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057420) roll requested 2024-11-11T16:37:37,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057427 2024-11-11T16:37:37,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057427 2024-11-11T16:37:37,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057427) roll requested 2024-11-11T16:37:37,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057433 2024-11-11T16:37:37,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057433 2024-11-11T16:37:37,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057433) roll requested 2024-11-11T16:37:37,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057439 2024-11-11T16:37:37,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057439 2024-11-11T16:37:37,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057439) roll requested 2024-11-11T16:37:37,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057446 2024-11-11T16:37:37,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057446 2024-11-11T16:37:37,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057446) roll requested 2024-11-11T16:37:37,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057453 2024-11-11T16:37:37,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057453 2024-11-11T16:37:37,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057453) roll requested 2024-11-11T16:37:37,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057459 2024-11-11T16:37:37,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057459 2024-11-11T16:37:37,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057459) roll requested 2024-11-11T16:37:37,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057467 2024-11-11T16:37:37,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057467 2024-11-11T16:37:37,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057467) roll requested 2024-11-11T16:37:37,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057477 2024-11-11T16:37:37,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057477 2024-11-11T16:37:37,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057477) roll requested 2024-11-11T16:37:37,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057489 2024-11-11T16:37:37,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057489 2024-11-11T16:37:37,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057489) roll requested 2024-11-11T16:37:37,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057501 2024-11-11T16:37:37,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057501 2024-11-11T16:37:37,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057501) roll requested 2024-11-11T16:37:37,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057508 2024-11-11T16:37:37,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057508 2024-11-11T16:37:37,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057508) roll requested 2024-11-11T16:37:37,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057515 2024-11-11T16:37:37,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057515 2024-11-11T16:37:37,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057515) roll requested 2024-11-11T16:37:37,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057521 2024-11-11T16:37:37,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057521 2024-11-11T16:37:37,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057521) roll requested 2024-11-11T16:37:37,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057528 2024-11-11T16:37:37,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057528 2024-11-11T16:37:37,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057528) roll requested 2024-11-11T16:37:37,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057535 2024-11-11T16:37:37,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057535 2024-11-11T16:37:37,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057535) roll requested 2024-11-11T16:37:37,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057543 2024-11-11T16:37:37,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057543 2024-11-11T16:37:37,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057543) roll requested 2024-11-11T16:37:37,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057551 2024-11-11T16:37:37,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057551 2024-11-11T16:37:37,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057551) roll requested 2024-11-11T16:37:37,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057564 2024-11-11T16:37:37,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057564 2024-11-11T16:37:37,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057564) roll requested 2024-11-11T16:37:37,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057574 2024-11-11T16:37:37,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057574 2024-11-11T16:37:37,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057574) roll requested 2024-11-11T16:37:37,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057584 2024-11-11T16:37:37,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057584 2024-11-11T16:37:37,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057584) roll requested 2024-11-11T16:37:37,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057592 2024-11-11T16:37:37,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057592 2024-11-11T16:37:37,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057592) roll requested 2024-11-11T16:37:37,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057599 2024-11-11T16:37:37,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057599 2024-11-11T16:37:37,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057599) roll requested 2024-11-11T16:37:37,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057605 2024-11-11T16:37:37,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057605 2024-11-11T16:37:37,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057605) roll requested 2024-11-11T16:37:37,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057612 2024-11-11T16:37:37,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057612 2024-11-11T16:37:37,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057612) roll requested 2024-11-11T16:37:37,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057619 2024-11-11T16:37:37,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057619 2024-11-11T16:37:37,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057619) roll requested 2024-11-11T16:37:37,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057625 2024-11-11T16:37:37,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057625 2024-11-11T16:37:37,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057625) roll requested 2024-11-11T16:37:37,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057632 2024-11-11T16:37:37,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057632 2024-11-11T16:37:37,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057632) roll requested 2024-11-11T16:37:37,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057638 2024-11-11T16:37:37,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057638 2024-11-11T16:37:37,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057638) roll requested 2024-11-11T16:37:37,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057645 2024-11-11T16:37:37,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057645 2024-11-11T16:37:37,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057645) roll requested 2024-11-11T16:37:37,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057652 2024-11-11T16:37:37,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057652 2024-11-11T16:37:37,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057652) roll requested 2024-11-11T16:37:37,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057659 2024-11-11T16:37:37,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057659 2024-11-11T16:37:37,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057659) roll requested 2024-11-11T16:37:37,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057665 2024-11-11T16:37:37,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057665 2024-11-11T16:37:37,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057665) roll requested 2024-11-11T16:37:37,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057672 2024-11-11T16:37:37,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057672 2024-11-11T16:37:37,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057672) roll requested 2024-11-11T16:37:37,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057679 2024-11-11T16:37:37,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057679 2024-11-11T16:37:37,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057679) roll requested 2024-11-11T16:37:37,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057685 2024-11-11T16:37:37,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057685 2024-11-11T16:37:37,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057685) roll requested 2024-11-11T16:37:37,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057692 2024-11-11T16:37:37,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057692 2024-11-11T16:37:37,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057692) roll requested 2024-11-11T16:37:37,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057698 2024-11-11T16:37:37,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057698 2024-11-11T16:37:37,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057698) roll requested 2024-11-11T16:37:37,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057705 2024-11-11T16:37:37,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057705 2024-11-11T16:37:37,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057705) roll requested 2024-11-11T16:37:37,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057711 2024-11-11T16:37:37,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057711 2024-11-11T16:37:37,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057711) roll requested 2024-11-11T16:37:37,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057718 2024-11-11T16:37:37,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057718 2024-11-11T16:37:37,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057718) roll requested 2024-11-11T16:37:37,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057724 2024-11-11T16:37:37,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057724 2024-11-11T16:37:37,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057724) roll requested 2024-11-11T16:37:37,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057731 2024-11-11T16:37:37,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057731 2024-11-11T16:37:37,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057731) roll requested 2024-11-11T16:37:37,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057738 2024-11-11T16:37:37,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057738 2024-11-11T16:37:37,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057738) roll requested 2024-11-11T16:37:37,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057744 2024-11-11T16:37:37,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057744 2024-11-11T16:37:37,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057744) roll requested 2024-11-11T16:37:37,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057751 2024-11-11T16:37:37,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057751 2024-11-11T16:37:37,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057751) roll requested 2024-11-11T16:37:37,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057757 2024-11-11T16:37:37,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057757 2024-11-11T16:37:37,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057757) roll requested 2024-11-11T16:37:37,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057764 2024-11-11T16:37:37,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057764 2024-11-11T16:37:37,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057764) roll requested 2024-11-11T16:37:37,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057771 2024-11-11T16:37:37,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057771 2024-11-11T16:37:37,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057771) roll requested 2024-11-11T16:37:37,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057777 2024-11-11T16:37:37,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057777 2024-11-11T16:37:37,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057777) roll requested 2024-11-11T16:37:37,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057784 2024-11-11T16:37:37,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057784 2024-11-11T16:37:37,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057784) roll requested 2024-11-11T16:37:37,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057790 2024-11-11T16:37:37,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057790 2024-11-11T16:37:37,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057790) roll requested 2024-11-11T16:37:37,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057797 2024-11-11T16:37:37,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057797 2024-11-11T16:37:37,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057797) roll requested 2024-11-11T16:37:37,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057805 2024-11-11T16:37:37,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057805 2024-11-11T16:37:37,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057805) roll requested 2024-11-11T16:37:37,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057812 2024-11-11T16:37:37,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057812 2024-11-11T16:37:37,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057812) roll requested 2024-11-11T16:37:37,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057818 2024-11-11T16:37:37,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057818 2024-11-11T16:37:37,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057818) roll requested 2024-11-11T16:37:37,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057827 2024-11-11T16:37:37,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057827 2024-11-11T16:37:37,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057827) roll requested 2024-11-11T16:37:37,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057834 2024-11-11T16:37:37,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057834 2024-11-11T16:37:37,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057834) roll requested 2024-11-11T16:37:37,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057840 2024-11-11T16:37:37,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057840 2024-11-11T16:37:37,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057840) roll requested 2024-11-11T16:37:37,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057847 2024-11-11T16:37:37,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057847 2024-11-11T16:37:37,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057847) roll requested 2024-11-11T16:37:37,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057853 2024-11-11T16:37:37,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057853 2024-11-11T16:37:37,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057853) roll requested 2024-11-11T16:37:37,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057860 2024-11-11T16:37:37,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057860 2024-11-11T16:37:37,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057860) roll requested 2024-11-11T16:37:37,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057866 2024-11-11T16:37:37,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057866 2024-11-11T16:37:37,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057866) roll requested 2024-11-11T16:37:37,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057874 2024-11-11T16:37:37,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057874 2024-11-11T16:37:37,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057874) roll requested 2024-11-11T16:37:37,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057888 2024-11-11T16:37:37,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057888 2024-11-11T16:37:37,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057888) roll requested 2024-11-11T16:37:37,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057894 2024-11-11T16:37:37,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057894 2024-11-11T16:37:37,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057894) roll requested 2024-11-11T16:37:37,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057901 2024-11-11T16:37:37,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057901 2024-11-11T16:37:37,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057901) roll requested 2024-11-11T16:37:37,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057908 2024-11-11T16:37:37,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057908 2024-11-11T16:37:37,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057908) roll requested 2024-11-11T16:37:37,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057918 2024-11-11T16:37:37,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057918 2024-11-11T16:37:37,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057918) roll requested 2024-11-11T16:37:37,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057925 2024-11-11T16:37:37,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057925 2024-11-11T16:37:37,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057925) roll requested 2024-11-11T16:37:37,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057932 2024-11-11T16:37:37,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057932 2024-11-11T16:37:37,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057932) roll requested 2024-11-11T16:37:37,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057938 2024-11-11T16:37:37,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057938 2024-11-11T16:37:37,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057938) roll requested 2024-11-11T16:37:37,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057945 2024-11-11T16:37:37,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057945 2024-11-11T16:37:37,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057945) roll requested 2024-11-11T16:37:37,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057951 2024-11-11T16:37:37,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057951 2024-11-11T16:37:37,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057951) roll requested 2024-11-11T16:37:37,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057958 2024-11-11T16:37:37,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057958 2024-11-11T16:37:37,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057958) roll requested 2024-11-11T16:37:37,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057964 2024-11-11T16:37:37,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057964 2024-11-11T16:37:37,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057964) roll requested 2024-11-11T16:37:37,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057971 2024-11-11T16:37:37,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057971 2024-11-11T16:37:37,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057971) roll requested 2024-11-11T16:37:37,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057977 2024-11-11T16:37:37,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057977 2024-11-11T16:37:37,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057977) roll requested 2024-11-11T16:37:37,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057983 2024-11-11T16:37:37,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057983 2024-11-11T16:37:37,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057983) roll requested 2024-11-11T16:37:37,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057990 2024-11-11T16:37:37,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:37,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057990 2024-11-11T16:37:37,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:37,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:37,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057990) roll requested 2024-11-11T16:37:37,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343057997 2024-11-11T16:37:38,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057997 2024-11-11T16:37:38,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343057997) roll requested 2024-11-11T16:37:38,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058006 2024-11-11T16:37:38,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343057997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058006 2024-11-11T16:37:38,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058006) roll requested 2024-11-11T16:37:38,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058013 2024-11-11T16:37:38,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058013 2024-11-11T16:37:38,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058013) roll requested 2024-11-11T16:37:38,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058020 2024-11-11T16:37:38,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058020 2024-11-11T16:37:38,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058020) roll requested 2024-11-11T16:37:38,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058027 2024-11-11T16:37:38,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058027 2024-11-11T16:37:38,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058027) roll requested 2024-11-11T16:37:38,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058034 2024-11-11T16:37:38,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058034 2024-11-11T16:37:38,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058034) roll requested 2024-11-11T16:37:38,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058040 2024-11-11T16:37:38,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058040 2024-11-11T16:37:38,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058040) roll requested 2024-11-11T16:37:38,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058049 2024-11-11T16:37:38,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058049 2024-11-11T16:37:38,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058049) roll requested 2024-11-11T16:37:38,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058060 2024-11-11T16:37:38,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058060 2024-11-11T16:37:38,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058060) roll requested 2024-11-11T16:37:38,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058067 2024-11-11T16:37:38,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058067 2024-11-11T16:37:38,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058067) roll requested 2024-11-11T16:37:38,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058074 2024-11-11T16:37:38,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058074 2024-11-11T16:37:38,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058074) roll requested 2024-11-11T16:37:38,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058080 2024-11-11T16:37:38,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058080 2024-11-11T16:37:38,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058080) roll requested 2024-11-11T16:37:38,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058087 2024-11-11T16:37:38,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058087 2024-11-11T16:37:38,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058087) roll requested 2024-11-11T16:37:38,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058095 2024-11-11T16:37:38,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058095 2024-11-11T16:37:38,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058095) roll requested 2024-11-11T16:37:38,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058101 2024-11-11T16:37:38,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058101 2024-11-11T16:37:38,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058101) roll requested 2024-11-11T16:37:38,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058108 2024-11-11T16:37:38,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058108 2024-11-11T16:37:38,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058108) roll requested 2024-11-11T16:37:38,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058114 2024-11-11T16:37:38,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058114 2024-11-11T16:37:38,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058114) roll requested 2024-11-11T16:37:38,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058121 2024-11-11T16:37:38,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058121 2024-11-11T16:37:38,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058121) roll requested 2024-11-11T16:37:38,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058127 2024-11-11T16:37:38,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058127 2024-11-11T16:37:38,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058127) roll requested 2024-11-11T16:37:38,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058133 2024-11-11T16:37:38,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058133 2024-11-11T16:37:38,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058133) roll requested 2024-11-11T16:37:38,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058140 2024-11-11T16:37:38,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058140 2024-11-11T16:37:38,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058140) roll requested 2024-11-11T16:37:38,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058147 2024-11-11T16:37:38,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058147 2024-11-11T16:37:38,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058147) roll requested 2024-11-11T16:37:38,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058153 2024-11-11T16:37:38,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058153 2024-11-11T16:37:38,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058153) roll requested 2024-11-11T16:37:38,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058161 2024-11-11T16:37:38,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058161 2024-11-11T16:37:38,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058161) roll requested 2024-11-11T16:37:38,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058168 2024-11-11T16:37:38,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058168 2024-11-11T16:37:38,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058168) roll requested 2024-11-11T16:37:38,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058175 2024-11-11T16:37:38,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058175 2024-11-11T16:37:38,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058175) roll requested 2024-11-11T16:37:38,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058182 2024-11-11T16:37:38,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058182 2024-11-11T16:37:38,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058182) roll requested 2024-11-11T16:37:38,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058188 2024-11-11T16:37:38,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058188 2024-11-11T16:37:38,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058188) roll requested 2024-11-11T16:37:38,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058194 2024-11-11T16:37:38,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058194 2024-11-11T16:37:38,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058194) roll requested 2024-11-11T16:37:38,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058201 2024-11-11T16:37:38,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058201 2024-11-11T16:37:38,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058201) roll requested 2024-11-11T16:37:38,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058216 2024-11-11T16:37:38,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058216 2024-11-11T16:37:38,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058216) roll requested 2024-11-11T16:37:38,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058223 2024-11-11T16:37:38,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058223 2024-11-11T16:37:38,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058223) roll requested 2024-11-11T16:37:38,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058232 2024-11-11T16:37:38,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058232 2024-11-11T16:37:38,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058232) roll requested 2024-11-11T16:37:38,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058239 2024-11-11T16:37:38,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058239 2024-11-11T16:37:38,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058239) roll requested 2024-11-11T16:37:38,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058246 2024-11-11T16:37:38,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058246 2024-11-11T16:37:38,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058246) roll requested 2024-11-11T16:37:38,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058255 2024-11-11T16:37:38,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058255 2024-11-11T16:37:38,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058255) roll requested 2024-11-11T16:37:38,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058262 2024-11-11T16:37:38,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058262 2024-11-11T16:37:38,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058262) roll requested 2024-11-11T16:37:38,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058270 2024-11-11T16:37:38,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058270 2024-11-11T16:37:38,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058270) roll requested 2024-11-11T16:37:38,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058282 2024-11-11T16:37:38,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058282 2024-11-11T16:37:38,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058282) roll requested 2024-11-11T16:37:38,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058297 2024-11-11T16:37:38,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058297 2024-11-11T16:37:38,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058297) roll requested 2024-11-11T16:37:38,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058309 2024-11-11T16:37:38,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058309 2024-11-11T16:37:38,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058309) roll requested 2024-11-11T16:37:38,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058316 2024-11-11T16:37:38,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058316 2024-11-11T16:37:38,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058316) roll requested 2024-11-11T16:37:38,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058322 2024-11-11T16:37:38,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058322 2024-11-11T16:37:38,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058322) roll requested 2024-11-11T16:37:38,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058329 2024-11-11T16:37:38,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058329 2024-11-11T16:37:38,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058329) roll requested 2024-11-11T16:37:38,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058339 2024-11-11T16:37:38,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058339 2024-11-11T16:37:38,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058339) roll requested 2024-11-11T16:37:38,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058348 2024-11-11T16:37:38,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058348 2024-11-11T16:37:38,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058348) roll requested 2024-11-11T16:37:38,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058356 2024-11-11T16:37:38,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058356 2024-11-11T16:37:38,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058356) roll requested 2024-11-11T16:37:38,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058363 2024-11-11T16:37:38,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058363 2024-11-11T16:37:38,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058363) roll requested 2024-11-11T16:37:38,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058370 2024-11-11T16:37:38,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058370 2024-11-11T16:37:38,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058370) roll requested 2024-11-11T16:37:38,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058376 2024-11-11T16:37:38,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058376 2024-11-11T16:37:38,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058376) roll requested 2024-11-11T16:37:38,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058383 2024-11-11T16:37:38,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058383 2024-11-11T16:37:38,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058383) roll requested 2024-11-11T16:37:38,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058389 2024-11-11T16:37:38,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058389 2024-11-11T16:37:38,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058389) roll requested 2024-11-11T16:37:38,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058397 2024-11-11T16:37:38,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058397 2024-11-11T16:37:38,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058397) roll requested 2024-11-11T16:37:38,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058413 2024-11-11T16:37:38,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058413 2024-11-11T16:37:38,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058413) roll requested 2024-11-11T16:37:38,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058421 2024-11-11T16:37:38,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058421 2024-11-11T16:37:38,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058421) roll requested 2024-11-11T16:37:38,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058428 2024-11-11T16:37:38,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058428 2024-11-11T16:37:38,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058428) roll requested 2024-11-11T16:37:38,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058437 2024-11-11T16:37:38,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058437 2024-11-11T16:37:38,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058437) roll requested 2024-11-11T16:37:38,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058444 2024-11-11T16:37:38,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058444 2024-11-11T16:37:38,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058444) roll requested 2024-11-11T16:37:38,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058452 2024-11-11T16:37:38,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058452 2024-11-11T16:37:38,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058452) roll requested 2024-11-11T16:37:38,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058459 2024-11-11T16:37:38,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058459 2024-11-11T16:37:38,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058459) roll requested 2024-11-11T16:37:38,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058466 2024-11-11T16:37:38,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058466 2024-11-11T16:37:38,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058466) roll requested 2024-11-11T16:37:38,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058473 2024-11-11T16:37:38,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058473 2024-11-11T16:37:38,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058473) roll requested 2024-11-11T16:37:38,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058480 2024-11-11T16:37:38,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058480 2024-11-11T16:37:38,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058480) roll requested 2024-11-11T16:37:38,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058489 2024-11-11T16:37:38,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058489 2024-11-11T16:37:38,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058489) roll requested 2024-11-11T16:37:38,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058496 2024-11-11T16:37:38,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058496 2024-11-11T16:37:38,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058496) roll requested 2024-11-11T16:37:38,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058502 2024-11-11T16:37:38,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058502 2024-11-11T16:37:38,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058502) roll requested 2024-11-11T16:37:38,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058508 2024-11-11T16:37:38,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058508 2024-11-11T16:37:38,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058508) roll requested 2024-11-11T16:37:38,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058515 2024-11-11T16:37:38,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058515 2024-11-11T16:37:38,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058515) roll requested 2024-11-11T16:37:38,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058521 2024-11-11T16:37:38,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058521 2024-11-11T16:37:38,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058521) roll requested 2024-11-11T16:37:38,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058527 2024-11-11T16:37:38,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058527 2024-11-11T16:37:38,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058527) roll requested 2024-11-11T16:37:38,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058537 2024-11-11T16:37:38,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058537 2024-11-11T16:37:38,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058537) roll requested 2024-11-11T16:37:38,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058544 2024-11-11T16:37:38,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058544 2024-11-11T16:37:38,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058544) roll requested 2024-11-11T16:37:38,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058550 2024-11-11T16:37:38,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058550 2024-11-11T16:37:38,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058550) roll requested 2024-11-11T16:37:38,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058558 2024-11-11T16:37:38,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058558 2024-11-11T16:37:38,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058558) roll requested 2024-11-11T16:37:38,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058564 2024-11-11T16:37:38,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058564 2024-11-11T16:37:38,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058564) roll requested 2024-11-11T16:37:38,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058571 2024-11-11T16:37:38,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058571 2024-11-11T16:37:38,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058571) roll requested 2024-11-11T16:37:38,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058578 2024-11-11T16:37:38,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058578 2024-11-11T16:37:38,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058578) roll requested 2024-11-11T16:37:38,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058584 2024-11-11T16:37:38,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058584 2024-11-11T16:37:38,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058584) roll requested 2024-11-11T16:37:38,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058591 2024-11-11T16:37:38,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058591 2024-11-11T16:37:38,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058591) roll requested 2024-11-11T16:37:38,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058598 2024-11-11T16:37:38,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058598 2024-11-11T16:37:38,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058598) roll requested 2024-11-11T16:37:38,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058605 2024-11-11T16:37:38,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058605 2024-11-11T16:37:38,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058605) roll requested 2024-11-11T16:37:38,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058611 2024-11-11T16:37:38,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058611 2024-11-11T16:37:38,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058611) roll requested 2024-11-11T16:37:38,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058618 2024-11-11T16:37:38,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058618 2024-11-11T16:37:38,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058618) roll requested 2024-11-11T16:37:38,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058624 2024-11-11T16:37:38,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058624 2024-11-11T16:37:38,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058624) roll requested 2024-11-11T16:37:38,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058630 2024-11-11T16:37:38,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058630 2024-11-11T16:37:38,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058630) roll requested 2024-11-11T16:37:38,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058637 2024-11-11T16:37:38,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058637 2024-11-11T16:37:38,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058637) roll requested 2024-11-11T16:37:38,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058643 2024-11-11T16:37:38,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058643 2024-11-11T16:37:38,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058643) roll requested 2024-11-11T16:37:38,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058650 2024-11-11T16:37:38,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058650 2024-11-11T16:37:38,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058650) roll requested 2024-11-11T16:37:38,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058657 2024-11-11T16:37:38,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058657 2024-11-11T16:37:38,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058657) roll requested 2024-11-11T16:37:38,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058663 2024-11-11T16:37:38,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058663 2024-11-11T16:37:38,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058663) roll requested 2024-11-11T16:37:38,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058670 2024-11-11T16:37:38,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058670 2024-11-11T16:37:38,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058670) roll requested 2024-11-11T16:37:38,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058677 2024-11-11T16:37:38,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058677 2024-11-11T16:37:38,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058677) roll requested 2024-11-11T16:37:38,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058684 2024-11-11T16:37:38,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058684 2024-11-11T16:37:38,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058684) roll requested 2024-11-11T16:37:38,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058692 2024-11-11T16:37:38,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058692 2024-11-11T16:37:38,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058692) roll requested 2024-11-11T16:37:38,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058698 2024-11-11T16:37:38,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058698 2024-11-11T16:37:38,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058698) roll requested 2024-11-11T16:37:38,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058706 2024-11-11T16:37:38,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058706 2024-11-11T16:37:38,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058706) roll requested 2024-11-11T16:37:38,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058729 2024-11-11T16:37:38,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058729 2024-11-11T16:37:38,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058729) roll requested 2024-11-11T16:37:38,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058742 2024-11-11T16:37:38,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058742 2024-11-11T16:37:38,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058742) roll requested 2024-11-11T16:37:38,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058749 2024-11-11T16:37:38,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058749 2024-11-11T16:37:38,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058749) roll requested 2024-11-11T16:37:38,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058755 2024-11-11T16:37:38,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058755 2024-11-11T16:37:38,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058755) roll requested 2024-11-11T16:37:38,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058762 2024-11-11T16:37:38,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058762 2024-11-11T16:37:38,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058762) roll requested 2024-11-11T16:37:38,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058768 2024-11-11T16:37:38,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058768 2024-11-11T16:37:38,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058768) roll requested 2024-11-11T16:37:38,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058774 2024-11-11T16:37:38,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058774 2024-11-11T16:37:38,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058774) roll requested 2024-11-11T16:37:38,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058781 2024-11-11T16:37:38,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058781 2024-11-11T16:37:38,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058781) roll requested 2024-11-11T16:37:38,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058787 2024-11-11T16:37:38,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058787 2024-11-11T16:37:38,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058787) roll requested 2024-11-11T16:37:38,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058796 2024-11-11T16:37:38,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058796 2024-11-11T16:37:38,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058796) roll requested 2024-11-11T16:37:38,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058810 2024-11-11T16:37:38,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058810 2024-11-11T16:37:38,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058810) roll requested 2024-11-11T16:37:38,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058817 2024-11-11T16:37:38,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058817 2024-11-11T16:37:38,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058817) roll requested 2024-11-11T16:37:38,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058823 2024-11-11T16:37:38,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058823 2024-11-11T16:37:38,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058823) roll requested 2024-11-11T16:37:38,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058829 2024-11-11T16:37:38,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058829 2024-11-11T16:37:38,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058829) roll requested 2024-11-11T16:37:38,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058836 2024-11-11T16:37:38,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058836 2024-11-11T16:37:38,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058836) roll requested 2024-11-11T16:37:38,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058842 2024-11-11T16:37:38,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058842 2024-11-11T16:37:38,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058842) roll requested 2024-11-11T16:37:38,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058849 2024-11-11T16:37:38,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058849 2024-11-11T16:37:38,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058849) roll requested 2024-11-11T16:37:38,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058855 2024-11-11T16:37:38,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058855 2024-11-11T16:37:38,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058855) roll requested 2024-11-11T16:37:38,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058862 2024-11-11T16:37:38,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058862 2024-11-11T16:37:38,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058862) roll requested 2024-11-11T16:37:38,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058869 2024-11-11T16:37:38,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058869 2024-11-11T16:37:38,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058869) roll requested 2024-11-11T16:37:38,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058876 2024-11-11T16:37:38,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058876 2024-11-11T16:37:38,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058876) roll requested 2024-11-11T16:37:38,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058894 2024-11-11T16:37:38,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058894 2024-11-11T16:37:38,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058894) roll requested 2024-11-11T16:37:38,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058903 2024-11-11T16:37:38,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058903 2024-11-11T16:37:38,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058903) roll requested 2024-11-11T16:37:38,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058911 2024-11-11T16:37:38,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058911 2024-11-11T16:37:38,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058911) roll requested 2024-11-11T16:37:38,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058918 2024-11-11T16:37:38,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058918 2024-11-11T16:37:38,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058918) roll requested 2024-11-11T16:37:38,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058925 2024-11-11T16:37:38,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058925 2024-11-11T16:37:38,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058925) roll requested 2024-11-11T16:37:38,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058932 2024-11-11T16:37:38,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058932 2024-11-11T16:37:38,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058932) roll requested 2024-11-11T16:37:38,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058938 2024-11-11T16:37:38,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058938 2024-11-11T16:37:38,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058938) roll requested 2024-11-11T16:37:38,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058945 2024-11-11T16:37:38,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058945 2024-11-11T16:37:38,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058945) roll requested 2024-11-11T16:37:38,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058951 2024-11-11T16:37:38,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058951 2024-11-11T16:37:38,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058951) roll requested 2024-11-11T16:37:38,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058958 2024-11-11T16:37:38,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058958 2024-11-11T16:37:38,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058958) roll requested 2024-11-11T16:37:38,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058964 2024-11-11T16:37:38,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058964 2024-11-11T16:37:38,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058964) roll requested 2024-11-11T16:37:38,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058971 2024-11-11T16:37:38,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058971 2024-11-11T16:37:38,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058971) roll requested 2024-11-11T16:37:38,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058977 2024-11-11T16:37:38,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058977 2024-11-11T16:37:38,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058977) roll requested 2024-11-11T16:37:38,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058984 2024-11-11T16:37:38,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058984 2024-11-11T16:37:38,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058984) roll requested 2024-11-11T16:37:38,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058990 2024-11-11T16:37:38,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:38,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058990 2024-11-11T16:37:38,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:38,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:38,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058990) roll requested 2024-11-11T16:37:38,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343058997 2024-11-11T16:37:39,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058997 2024-11-11T16:37:39,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343058997) roll requested 2024-11-11T16:37:39,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059004 2024-11-11T16:37:39,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343058997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059004 2024-11-11T16:37:39,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059004) roll requested 2024-11-11T16:37:39,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059010 2024-11-11T16:37:39,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059010 2024-11-11T16:37:39,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059010) roll requested 2024-11-11T16:37:39,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059018 2024-11-11T16:37:39,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059018 2024-11-11T16:37:39,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059018) roll requested 2024-11-11T16:37:39,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059024 2024-11-11T16:37:39,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059024 2024-11-11T16:37:39,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059024) roll requested 2024-11-11T16:37:39,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059032 2024-11-11T16:37:39,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059032 2024-11-11T16:37:39,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059032) roll requested 2024-11-11T16:37:39,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059039 2024-11-11T16:37:39,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059039 2024-11-11T16:37:39,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059039) roll requested 2024-11-11T16:37:39,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059073 2024-11-11T16:37:39,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059073 2024-11-11T16:37:39,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059073) roll requested 2024-11-11T16:37:39,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059101 2024-11-11T16:37:39,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059101 2024-11-11T16:37:39,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059101) roll requested 2024-11-11T16:37:39,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059110 2024-11-11T16:37:39,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059110 2024-11-11T16:37:39,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059110) roll requested 2024-11-11T16:37:39,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059119 2024-11-11T16:37:39,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059119 2024-11-11T16:37:39,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059119) roll requested 2024-11-11T16:37:39,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059129 2024-11-11T16:37:39,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059129 2024-11-11T16:37:39,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059129) roll requested 2024-11-11T16:37:39,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059137 2024-11-11T16:37:39,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059137 2024-11-11T16:37:39,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059137) roll requested 2024-11-11T16:37:39,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059147 2024-11-11T16:37:39,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059147 2024-11-11T16:37:39,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059147) roll requested 2024-11-11T16:37:39,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059173 2024-11-11T16:37:39,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059173 2024-11-11T16:37:39,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059173) roll requested 2024-11-11T16:37:39,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059188 2024-11-11T16:37:39,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059188 2024-11-11T16:37:39,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059188) roll requested 2024-11-11T16:37:39,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059201 2024-11-11T16:37:39,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059201 2024-11-11T16:37:39,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059201) roll requested 2024-11-11T16:37:39,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059210 2024-11-11T16:37:39,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059210 2024-11-11T16:37:39,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059210) roll requested 2024-11-11T16:37:39,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059218 2024-11-11T16:37:39,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059218 2024-11-11T16:37:39,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059218) roll requested 2024-11-11T16:37:39,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059233 2024-11-11T16:37:39,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059233 2024-11-11T16:37:39,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059233) roll requested 2024-11-11T16:37:39,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059245 2024-11-11T16:37:39,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059245 2024-11-11T16:37:39,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059245) roll requested 2024-11-11T16:37:39,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059265 2024-11-11T16:37:39,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059265 2024-11-11T16:37:39,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059265) roll requested 2024-11-11T16:37:39,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059277 2024-11-11T16:37:39,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059277 2024-11-11T16:37:39,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059277) roll requested 2024-11-11T16:37:39,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059285 2024-11-11T16:37:39,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059285 2024-11-11T16:37:39,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059285) roll requested 2024-11-11T16:37:39,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059294 2024-11-11T16:37:39,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059294 2024-11-11T16:37:39,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059294) roll requested 2024-11-11T16:37:39,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059305 2024-11-11T16:37:39,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059305 2024-11-11T16:37:39,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059305) roll requested 2024-11-11T16:37:39,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059317 2024-11-11T16:37:39,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059317 2024-11-11T16:37:39,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059317) roll requested 2024-11-11T16:37:39,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059337 2024-11-11T16:37:39,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059337 2024-11-11T16:37:39,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059337) roll requested 2024-11-11T16:37:39,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059347 2024-11-11T16:37:39,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059347 2024-11-11T16:37:39,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059347) roll requested 2024-11-11T16:37:39,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059356 2024-11-11T16:37:39,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059356 2024-11-11T16:37:39,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059356) roll requested 2024-11-11T16:37:39,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059370 2024-11-11T16:37:39,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059370 2024-11-11T16:37:39,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059370) roll requested 2024-11-11T16:37:39,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059382 2024-11-11T16:37:39,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059382 2024-11-11T16:37:39,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059382) roll requested 2024-11-11T16:37:39,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059397 2024-11-11T16:37:39,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059397 2024-11-11T16:37:39,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059397) roll requested 2024-11-11T16:37:39,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059409 2024-11-11T16:37:39,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059409 2024-11-11T16:37:39,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059409) roll requested 2024-11-11T16:37:39,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059419 2024-11-11T16:37:39,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059419 2024-11-11T16:37:39,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059419) roll requested 2024-11-11T16:37:39,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059429 2024-11-11T16:37:39,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059429 2024-11-11T16:37:39,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059429) roll requested 2024-11-11T16:37:39,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059441 2024-11-11T16:37:39,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059441 2024-11-11T16:37:39,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059441) roll requested 2024-11-11T16:37:39,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059449 2024-11-11T16:37:39,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059449 2024-11-11T16:37:39,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059449) roll requested 2024-11-11T16:37:39,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059457 2024-11-11T16:37:39,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059457 2024-11-11T16:37:39,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059457) roll requested 2024-11-11T16:37:39,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059465 2024-11-11T16:37:39,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059465 2024-11-11T16:37:39,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059465) roll requested 2024-11-11T16:37:39,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059471 2024-11-11T16:37:39,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059471 2024-11-11T16:37:39,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059471) roll requested 2024-11-11T16:37:39,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059478 2024-11-11T16:37:39,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059478 2024-11-11T16:37:39,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059478) roll requested 2024-11-11T16:37:39,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059485 2024-11-11T16:37:39,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059485 2024-11-11T16:37:39,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059485) roll requested 2024-11-11T16:37:39,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059492 2024-11-11T16:37:39,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059492 2024-11-11T16:37:39,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059492) roll requested 2024-11-11T16:37:39,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059498 2024-11-11T16:37:39,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059498 2024-11-11T16:37:39,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059498) roll requested 2024-11-11T16:37:39,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059506 2024-11-11T16:37:39,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059506 2024-11-11T16:37:39,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059506) roll requested 2024-11-11T16:37:39,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059512 2024-11-11T16:37:39,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059512 2024-11-11T16:37:39,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059512) roll requested 2024-11-11T16:37:39,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059519 2024-11-11T16:37:39,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059519 2024-11-11T16:37:39,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059519) roll requested 2024-11-11T16:37:39,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059529 2024-11-11T16:37:39,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059529 2024-11-11T16:37:39,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059529) roll requested 2024-11-11T16:37:39,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059537 2024-11-11T16:37:39,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059537 2024-11-11T16:37:39,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059537) roll requested 2024-11-11T16:37:39,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059549 2024-11-11T16:37:39,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059549 2024-11-11T16:37:39,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059549) roll requested 2024-11-11T16:37:39,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059559 2024-11-11T16:37:39,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059559 2024-11-11T16:37:39,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059559) roll requested 2024-11-11T16:37:39,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059567 2024-11-11T16:37:39,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059567 2024-11-11T16:37:39,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059567) roll requested 2024-11-11T16:37:39,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059573 2024-11-11T16:37:39,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059573 2024-11-11T16:37:39,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059573) roll requested 2024-11-11T16:37:39,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059581 2024-11-11T16:37:39,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059581 2024-11-11T16:37:39,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059581) roll requested 2024-11-11T16:37:39,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059587 2024-11-11T16:37:39,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059587 2024-11-11T16:37:39,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059587) roll requested 2024-11-11T16:37:39,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059594 2024-11-11T16:37:39,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059594 2024-11-11T16:37:39,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059594) roll requested 2024-11-11T16:37:39,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059601 2024-11-11T16:37:39,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059601 2024-11-11T16:37:39,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059601) roll requested 2024-11-11T16:37:39,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059608 2024-11-11T16:37:39,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059608 2024-11-11T16:37:39,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059608) roll requested 2024-11-11T16:37:39,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059615 2024-11-11T16:37:39,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059615 2024-11-11T16:37:39,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059615) roll requested 2024-11-11T16:37:39,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059622 2024-11-11T16:37:39,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059622 2024-11-11T16:37:39,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059622) roll requested 2024-11-11T16:37:39,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059628 2024-11-11T16:37:39,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059628 2024-11-11T16:37:39,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059628) roll requested 2024-11-11T16:37:39,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059635 2024-11-11T16:37:39,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059635 2024-11-11T16:37:39,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059635) roll requested 2024-11-11T16:37:39,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059644 2024-11-11T16:37:39,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059644 2024-11-11T16:37:39,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059644) roll requested 2024-11-11T16:37:39,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059650 2024-11-11T16:37:39,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059650 2024-11-11T16:37:39,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059650) roll requested 2024-11-11T16:37:39,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059657 2024-11-11T16:37:39,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059657 2024-11-11T16:37:39,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059657) roll requested 2024-11-11T16:37:39,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059668 2024-11-11T16:37:39,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059668 2024-11-11T16:37:39,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059668) roll requested 2024-11-11T16:37:39,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059675 2024-11-11T16:37:39,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059675 2024-11-11T16:37:39,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059675) roll requested 2024-11-11T16:37:39,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059681 2024-11-11T16:37:39,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059681 2024-11-11T16:37:39,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059681) roll requested 2024-11-11T16:37:39,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059697 2024-11-11T16:37:39,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059697 2024-11-11T16:37:39,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059697) roll requested 2024-11-11T16:37:39,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059712 2024-11-11T16:37:39,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059712 2024-11-11T16:37:39,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059712) roll requested 2024-11-11T16:37:39,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059722 2024-11-11T16:37:39,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059722 2024-11-11T16:37:39,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059722) roll requested 2024-11-11T16:37:39,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059730 2024-11-11T16:37:39,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059730 2024-11-11T16:37:39,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059730) roll requested 2024-11-11T16:37:39,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059738 2024-11-11T16:37:39,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059738 2024-11-11T16:37:39,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059738) roll requested 2024-11-11T16:37:39,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059746 2024-11-11T16:37:39,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059746 2024-11-11T16:37:39,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059746) roll requested 2024-11-11T16:37:39,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059755 2024-11-11T16:37:39,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059755 2024-11-11T16:37:39,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059755) roll requested 2024-11-11T16:37:39,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059763 2024-11-11T16:37:39,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059763 2024-11-11T16:37:39,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059763) roll requested 2024-11-11T16:37:39,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059770 2024-11-11T16:37:39,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059770 2024-11-11T16:37:39,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059770) roll requested 2024-11-11T16:37:39,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059778 2024-11-11T16:37:39,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059778 2024-11-11T16:37:39,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059778) roll requested 2024-11-11T16:37:39,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059784 2024-11-11T16:37:39,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059784 2024-11-11T16:37:39,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059784) roll requested 2024-11-11T16:37:39,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059791 2024-11-11T16:37:39,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059791 2024-11-11T16:37:39,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059791) roll requested 2024-11-11T16:37:39,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059798 2024-11-11T16:37:39,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059798 2024-11-11T16:37:39,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059798) roll requested 2024-11-11T16:37:39,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059805 2024-11-11T16:37:39,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059805 2024-11-11T16:37:39,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059805) roll requested 2024-11-11T16:37:39,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059811 2024-11-11T16:37:39,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059811 2024-11-11T16:37:39,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059811) roll requested 2024-11-11T16:37:39,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059818 2024-11-11T16:37:39,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059818 2024-11-11T16:37:39,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059818) roll requested 2024-11-11T16:37:39,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059827 2024-11-11T16:37:39,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059827 2024-11-11T16:37:39,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059827) roll requested 2024-11-11T16:37:39,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059834 2024-11-11T16:37:39,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059834 2024-11-11T16:37:39,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059834) roll requested 2024-11-11T16:37:39,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059841 2024-11-11T16:37:39,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059841 2024-11-11T16:37:39,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059841) roll requested 2024-11-11T16:37:39,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059847 2024-11-11T16:37:39,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059847 2024-11-11T16:37:39,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059847) roll requested 2024-11-11T16:37:39,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059854 2024-11-11T16:37:39,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059854 2024-11-11T16:37:39,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059854) roll requested 2024-11-11T16:37:39,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059861 2024-11-11T16:37:39,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059861 2024-11-11T16:37:39,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059861) roll requested 2024-11-11T16:37:39,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059867 2024-11-11T16:37:39,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059867 2024-11-11T16:37:39,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059867) roll requested 2024-11-11T16:37:39,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059874 2024-11-11T16:37:39,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059874 2024-11-11T16:37:39,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059874) roll requested 2024-11-11T16:37:39,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059881 2024-11-11T16:37:39,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059881 2024-11-11T16:37:39,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059881) roll requested 2024-11-11T16:37:39,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059887 2024-11-11T16:37:39,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059887 2024-11-11T16:37:39,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059887) roll requested 2024-11-11T16:37:39,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059894 2024-11-11T16:37:39,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059894 2024-11-11T16:37:39,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059894) roll requested 2024-11-11T16:37:39,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059900 2024-11-11T16:37:39,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059900 2024-11-11T16:37:39,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059900) roll requested 2024-11-11T16:37:39,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059907 2024-11-11T16:37:39,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059907 2024-11-11T16:37:39,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059907) roll requested 2024-11-11T16:37:39,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059914 2024-11-11T16:37:39,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059914 2024-11-11T16:37:39,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059914) roll requested 2024-11-11T16:37:39,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059920 2024-11-11T16:37:39,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059920 2024-11-11T16:37:39,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059920) roll requested 2024-11-11T16:37:39,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059927 2024-11-11T16:37:39,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059927 2024-11-11T16:37:39,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059927) roll requested 2024-11-11T16:37:39,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059934 2024-11-11T16:37:39,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059934 2024-11-11T16:37:39,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059934) roll requested 2024-11-11T16:37:39,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059940 2024-11-11T16:37:39,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059940 2024-11-11T16:37:39,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059940) roll requested 2024-11-11T16:37:39,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059946 2024-11-11T16:37:39,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059946 2024-11-11T16:37:39,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059946) roll requested 2024-11-11T16:37:39,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059954 2024-11-11T16:37:39,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059954 2024-11-11T16:37:39,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059954) roll requested 2024-11-11T16:37:39,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059961 2024-11-11T16:37:39,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059961 2024-11-11T16:37:39,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059961) roll requested 2024-11-11T16:37:39,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059968 2024-11-11T16:37:39,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059968 2024-11-11T16:37:39,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059968) roll requested 2024-11-11T16:37:39,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059974 2024-11-11T16:37:39,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059974 2024-11-11T16:37:39,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059974) roll requested 2024-11-11T16:37:39,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059980 2024-11-11T16:37:39,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059980 2024-11-11T16:37:39,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059980) roll requested 2024-11-11T16:37:39,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059987 2024-11-11T16:37:39,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059987 2024-11-11T16:37:39,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059987) roll requested 2024-11-11T16:37:39,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343059993 2024-11-11T16:37:39,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:39,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059993 2024-11-11T16:37:39,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:39,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:39,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343059993) roll requested 2024-11-11T16:37:40,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060000 2024-11-11T16:37:40,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343059993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060000 2024-11-11T16:37:40,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060000) roll requested 2024-11-11T16:37:40,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060006 2024-11-11T16:37:40,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060006 2024-11-11T16:37:40,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060006) roll requested 2024-11-11T16:37:40,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060012 2024-11-11T16:37:40,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060012 2024-11-11T16:37:40,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060012) roll requested 2024-11-11T16:37:40,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060019 2024-11-11T16:37:40,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060019 2024-11-11T16:37:40,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060019) roll requested 2024-11-11T16:37:40,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060026 2024-11-11T16:37:40,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060026 2024-11-11T16:37:40,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060026) roll requested 2024-11-11T16:37:40,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060032 2024-11-11T16:37:40,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060032 2024-11-11T16:37:40,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060032) roll requested 2024-11-11T16:37:40,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060039 2024-11-11T16:37:40,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060039 2024-11-11T16:37:40,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060039) roll requested 2024-11-11T16:37:40,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060045 2024-11-11T16:37:40,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060045 2024-11-11T16:37:40,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060045) roll requested 2024-11-11T16:37:40,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060052 2024-11-11T16:37:40,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060052 2024-11-11T16:37:40,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060052) roll requested 2024-11-11T16:37:40,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060059 2024-11-11T16:37:40,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060059 2024-11-11T16:37:40,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060059) roll requested 2024-11-11T16:37:40,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060065 2024-11-11T16:37:40,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060065 2024-11-11T16:37:40,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060065) roll requested 2024-11-11T16:37:40,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060072 2024-11-11T16:37:40,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060072 2024-11-11T16:37:40,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060072) roll requested 2024-11-11T16:37:40,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060078 2024-11-11T16:37:40,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060078 2024-11-11T16:37:40,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060078) roll requested 2024-11-11T16:37:40,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060085 2024-11-11T16:37:40,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060085 2024-11-11T16:37:40,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060085) roll requested 2024-11-11T16:37:40,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060091 2024-11-11T16:37:40,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060091 2024-11-11T16:37:40,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060091) roll requested 2024-11-11T16:37:40,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060098 2024-11-11T16:37:40,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060098 2024-11-11T16:37:40,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060098) roll requested 2024-11-11T16:37:40,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060104 2024-11-11T16:37:40,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060104 2024-11-11T16:37:40,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060104) roll requested 2024-11-11T16:37:40,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060111 2024-11-11T16:37:40,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060111 2024-11-11T16:37:40,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060111) roll requested 2024-11-11T16:37:40,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060117 2024-11-11T16:37:40,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060117 2024-11-11T16:37:40,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060117) roll requested 2024-11-11T16:37:40,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060124 2024-11-11T16:37:40,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060124 2024-11-11T16:37:40,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060124) roll requested 2024-11-11T16:37:40,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060137 2024-11-11T16:37:40,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060137 2024-11-11T16:37:40,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060137) roll requested 2024-11-11T16:37:40,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060153 2024-11-11T16:37:40,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060153 2024-11-11T16:37:40,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060153) roll requested 2024-11-11T16:37:40,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060162 2024-11-11T16:37:40,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060162 2024-11-11T16:37:40,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060162) roll requested 2024-11-11T16:37:40,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060170 2024-11-11T16:37:40,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060170 2024-11-11T16:37:40,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060170) roll requested 2024-11-11T16:37:40,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060178 2024-11-11T16:37:40,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060178 2024-11-11T16:37:40,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060178) roll requested 2024-11-11T16:37:40,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060191 2024-11-11T16:37:40,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060191 2024-11-11T16:37:40,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060191) roll requested 2024-11-11T16:37:40,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060197 2024-11-11T16:37:40,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060197 2024-11-11T16:37:40,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060197) roll requested 2024-11-11T16:37:40,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060208 2024-11-11T16:37:40,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060208 2024-11-11T16:37:40,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060208) roll requested 2024-11-11T16:37:40,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060214 2024-11-11T16:37:40,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060214 2024-11-11T16:37:40,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060214) roll requested 2024-11-11T16:37:40,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060221 2024-11-11T16:37:40,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060221 2024-11-11T16:37:40,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060221) roll requested 2024-11-11T16:37:40,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060227 2024-11-11T16:37:40,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060227 2024-11-11T16:37:40,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060227) roll requested 2024-11-11T16:37:40,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060234 2024-11-11T16:37:40,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060234 2024-11-11T16:37:40,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060234) roll requested 2024-11-11T16:37:40,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060240 2024-11-11T16:37:40,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060240 2024-11-11T16:37:40,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060240) roll requested 2024-11-11T16:37:40,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060246 2024-11-11T16:37:40,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060246 2024-11-11T16:37:40,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060246) roll requested 2024-11-11T16:37:40,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060253 2024-11-11T16:37:40,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060253 2024-11-11T16:37:40,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060253) roll requested 2024-11-11T16:37:40,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060259 2024-11-11T16:37:40,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060259 2024-11-11T16:37:40,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060259) roll requested 2024-11-11T16:37:40,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060266 2024-11-11T16:37:40,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060266 2024-11-11T16:37:40,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060266) roll requested 2024-11-11T16:37:40,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060272 2024-11-11T16:37:40,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060272 2024-11-11T16:37:40,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060272) roll requested 2024-11-11T16:37:40,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060279 2024-11-11T16:37:40,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060279 2024-11-11T16:37:40,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060279) roll requested 2024-11-11T16:37:40,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060286 2024-11-11T16:37:40,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060286 2024-11-11T16:37:40,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060286) roll requested 2024-11-11T16:37:40,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060292 2024-11-11T16:37:40,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060292 2024-11-11T16:37:40,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060292) roll requested 2024-11-11T16:37:40,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060299 2024-11-11T16:37:40,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060299 2024-11-11T16:37:40,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060299) roll requested 2024-11-11T16:37:40,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060305 2024-11-11T16:37:40,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060305 2024-11-11T16:37:40,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060305) roll requested 2024-11-11T16:37:40,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060312 2024-11-11T16:37:40,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060312 2024-11-11T16:37:40,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060312) roll requested 2024-11-11T16:37:40,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060318 2024-11-11T16:37:40,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060318 2024-11-11T16:37:40,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060318) roll requested 2024-11-11T16:37:40,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060324 2024-11-11T16:37:40,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060324 2024-11-11T16:37:40,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060324) roll requested 2024-11-11T16:37:40,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060331 2024-11-11T16:37:40,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060331 2024-11-11T16:37:40,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060331) roll requested 2024-11-11T16:37:40,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060338 2024-11-11T16:37:40,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060338 2024-11-11T16:37:40,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060338) roll requested 2024-11-11T16:37:40,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060350 2024-11-11T16:37:40,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060350 2024-11-11T16:37:40,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060350) roll requested 2024-11-11T16:37:40,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060356 2024-11-11T16:37:40,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060356 2024-11-11T16:37:40,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060356) roll requested 2024-11-11T16:37:40,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060363 2024-11-11T16:37:40,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060363 2024-11-11T16:37:40,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060363) roll requested 2024-11-11T16:37:40,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060369 2024-11-11T16:37:40,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060369 2024-11-11T16:37:40,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060369) roll requested 2024-11-11T16:37:40,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060376 2024-11-11T16:37:40,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060376 2024-11-11T16:37:40,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060376) roll requested 2024-11-11T16:37:40,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060382 2024-11-11T16:37:40,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060382 2024-11-11T16:37:40,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060382) roll requested 2024-11-11T16:37:40,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060389 2024-11-11T16:37:40,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060389 2024-11-11T16:37:40,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060389) roll requested 2024-11-11T16:37:40,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060398 2024-11-11T16:37:40,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060398 2024-11-11T16:37:40,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060398) roll requested 2024-11-11T16:37:40,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060404 2024-11-11T16:37:40,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060404 2024-11-11T16:37:40,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060404) roll requested 2024-11-11T16:37:40,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060411 2024-11-11T16:37:40,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060411 2024-11-11T16:37:40,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060411) roll requested 2024-11-11T16:37:40,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060420 2024-11-11T16:37:40,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060420 2024-11-11T16:37:40,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060420) roll requested 2024-11-11T16:37:40,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060427 2024-11-11T16:37:40,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060427 2024-11-11T16:37:40,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060427) roll requested 2024-11-11T16:37:40,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060437 2024-11-11T16:37:40,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060437 2024-11-11T16:37:40,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060437) roll requested 2024-11-11T16:37:40,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060445 2024-11-11T16:37:40,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060445 2024-11-11T16:37:40,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060445) roll requested 2024-11-11T16:37:40,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060451 2024-11-11T16:37:40,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060451 2024-11-11T16:37:40,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060451) roll requested 2024-11-11T16:37:40,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060458 2024-11-11T16:37:40,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060458 2024-11-11T16:37:40,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060458) roll requested 2024-11-11T16:37:40,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060465 2024-11-11T16:37:40,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060465 2024-11-11T16:37:40,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060465) roll requested 2024-11-11T16:37:40,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060477 2024-11-11T16:37:40,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060477 2024-11-11T16:37:40,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060477) roll requested 2024-11-11T16:37:40,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060485 2024-11-11T16:37:40,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060485 2024-11-11T16:37:40,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060485) roll requested 2024-11-11T16:37:40,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060496 2024-11-11T16:37:40,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060496 2024-11-11T16:37:40,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060496) roll requested 2024-11-11T16:37:40,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060507 2024-11-11T16:37:40,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060507 2024-11-11T16:37:40,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060507) roll requested 2024-11-11T16:37:40,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060514 2024-11-11T16:37:40,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060514 2024-11-11T16:37:40,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060514) roll requested 2024-11-11T16:37:40,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060520 2024-11-11T16:37:40,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060520 2024-11-11T16:37:40,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060520) roll requested 2024-11-11T16:37:40,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060527 2024-11-11T16:37:40,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060527 2024-11-11T16:37:40,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060527) roll requested 2024-11-11T16:37:40,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060536 2024-11-11T16:37:40,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060536 2024-11-11T16:37:40,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060536) roll requested 2024-11-11T16:37:40,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060543 2024-11-11T16:37:40,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060543 2024-11-11T16:37:40,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060543) roll requested 2024-11-11T16:37:40,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060549 2024-11-11T16:37:40,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060549 2024-11-11T16:37:40,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060549) roll requested 2024-11-11T16:37:40,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060556 2024-11-11T16:37:40,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060556 2024-11-11T16:37:40,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060556) roll requested 2024-11-11T16:37:40,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060562 2024-11-11T16:37:40,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060562 2024-11-11T16:37:40,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060562) roll requested 2024-11-11T16:37:40,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060568 2024-11-11T16:37:40,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060568 2024-11-11T16:37:40,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060568) roll requested 2024-11-11T16:37:40,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060575 2024-11-11T16:37:40,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060575 2024-11-11T16:37:40,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060575) roll requested 2024-11-11T16:37:40,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060582 2024-11-11T16:37:40,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060582 2024-11-11T16:37:40,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060582) roll requested 2024-11-11T16:37:40,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060589 2024-11-11T16:37:40,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060589 2024-11-11T16:37:40,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060589) roll requested 2024-11-11T16:37:40,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060595 2024-11-11T16:37:40,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060595 2024-11-11T16:37:40,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060595) roll requested 2024-11-11T16:37:40,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060604 2024-11-11T16:37:40,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060604 2024-11-11T16:37:40,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060604) roll requested 2024-11-11T16:37:40,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060611 2024-11-11T16:37:40,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060611 2024-11-11T16:37:40,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060611) roll requested 2024-11-11T16:37:40,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060617 2024-11-11T16:37:40,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060617 2024-11-11T16:37:40,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060617) roll requested 2024-11-11T16:37:40,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060624 2024-11-11T16:37:40,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060624 2024-11-11T16:37:40,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060624) roll requested 2024-11-11T16:37:40,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060630 2024-11-11T16:37:40,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060630 2024-11-11T16:37:40,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060630) roll requested 2024-11-11T16:37:40,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060638 2024-11-11T16:37:40,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060638 2024-11-11T16:37:40,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060638) roll requested 2024-11-11T16:37:40,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060650 2024-11-11T16:37:40,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060650 2024-11-11T16:37:40,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060650) roll requested 2024-11-11T16:37:40,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060657 2024-11-11T16:37:40,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060657 2024-11-11T16:37:40,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060657) roll requested 2024-11-11T16:37:40,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060664 2024-11-11T16:37:40,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060664 2024-11-11T16:37:40,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060664) roll requested 2024-11-11T16:37:40,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060670 2024-11-11T16:37:40,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060670 2024-11-11T16:37:40,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060670) roll requested 2024-11-11T16:37:40,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060679 2024-11-11T16:37:40,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060679 2024-11-11T16:37:40,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060679) roll requested 2024-11-11T16:37:40,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060687 2024-11-11T16:37:40,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060687 2024-11-11T16:37:40,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060687) roll requested 2024-11-11T16:37:40,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060694 2024-11-11T16:37:40,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060694 2024-11-11T16:37:40,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060694) roll requested 2024-11-11T16:37:40,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060702 2024-11-11T16:37:40,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060702 2024-11-11T16:37:40,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060702) roll requested 2024-11-11T16:37:40,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060709 2024-11-11T16:37:40,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060709 2024-11-11T16:37:40,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060709) roll requested 2024-11-11T16:37:40,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060715 2024-11-11T16:37:40,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060715 2024-11-11T16:37:40,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060715) roll requested 2024-11-11T16:37:40,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060722 2024-11-11T16:37:40,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060722 2024-11-11T16:37:40,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060722) roll requested 2024-11-11T16:37:40,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060729 2024-11-11T16:37:40,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060729 2024-11-11T16:37:40,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060729) roll requested 2024-11-11T16:37:40,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060735 2024-11-11T16:37:40,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060735 2024-11-11T16:37:40,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060735) roll requested 2024-11-11T16:37:40,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060742 2024-11-11T16:37:40,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060742 2024-11-11T16:37:40,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060742) roll requested 2024-11-11T16:37:40,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060748 2024-11-11T16:37:40,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060748 2024-11-11T16:37:40,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060748) roll requested 2024-11-11T16:37:40,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060755 2024-11-11T16:37:40,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060755 2024-11-11T16:37:40,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060755) roll requested 2024-11-11T16:37:40,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060761 2024-11-11T16:37:40,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060761 2024-11-11T16:37:40,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060761) roll requested 2024-11-11T16:37:40,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060768 2024-11-11T16:37:40,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060768 2024-11-11T16:37:40,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060768) roll requested 2024-11-11T16:37:40,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060776 2024-11-11T16:37:40,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060776 2024-11-11T16:37:40,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060776) roll requested 2024-11-11T16:37:40,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060783 2024-11-11T16:37:40,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060783 2024-11-11T16:37:40,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060783) roll requested 2024-11-11T16:37:40,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060791 2024-11-11T16:37:40,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060791 2024-11-11T16:37:40,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060791) roll requested 2024-11-11T16:37:40,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060799 2024-11-11T16:37:40,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060799 2024-11-11T16:37:40,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060799) roll requested 2024-11-11T16:37:40,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060806 2024-11-11T16:37:40,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060806 2024-11-11T16:37:40,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060806) roll requested 2024-11-11T16:37:40,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060813 2024-11-11T16:37:40,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060813 2024-11-11T16:37:40,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060813) roll requested 2024-11-11T16:37:40,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060821 2024-11-11T16:37:40,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060821 2024-11-11T16:37:40,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060821) roll requested 2024-11-11T16:37:40,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060829 2024-11-11T16:37:40,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060829 2024-11-11T16:37:40,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060829) roll requested 2024-11-11T16:37:40,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060837 2024-11-11T16:37:40,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060837 2024-11-11T16:37:40,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060837) roll requested 2024-11-11T16:37:40,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060846 2024-11-11T16:37:40,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060846 2024-11-11T16:37:40,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060846) roll requested 2024-11-11T16:37:40,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060856 2024-11-11T16:37:40,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060856 2024-11-11T16:37:40,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060856) roll requested 2024-11-11T16:37:40,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060863 2024-11-11T16:37:40,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060863 2024-11-11T16:37:40,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060863) roll requested 2024-11-11T16:37:40,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060870 2024-11-11T16:37:40,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060870 2024-11-11T16:37:40,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060870) roll requested 2024-11-11T16:37:40,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060879 2024-11-11T16:37:40,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060879 2024-11-11T16:37:40,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060879) roll requested 2024-11-11T16:37:40,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060886 2024-11-11T16:37:40,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060886 2024-11-11T16:37:40,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060886) roll requested 2024-11-11T16:37:40,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060897 2024-11-11T16:37:40,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060897 2024-11-11T16:37:40,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060897) roll requested 2024-11-11T16:37:40,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060907 2024-11-11T16:37:40,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060907 2024-11-11T16:37:40,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060907) roll requested 2024-11-11T16:37:40,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060917 2024-11-11T16:37:40,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060917 2024-11-11T16:37:40,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060917) roll requested 2024-11-11T16:37:40,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060924 2024-11-11T16:37:40,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060924 2024-11-11T16:37:40,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060924) roll requested 2024-11-11T16:37:40,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060933 2024-11-11T16:37:40,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060933 2024-11-11T16:37:40,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060933) roll requested 2024-11-11T16:37:40,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060939 2024-11-11T16:37:40,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060939 2024-11-11T16:37:40,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060939) roll requested 2024-11-11T16:37:40,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060946 2024-11-11T16:37:40,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060946 2024-11-11T16:37:40,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060946) roll requested 2024-11-11T16:37:40,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060953 2024-11-11T16:37:40,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060953 2024-11-11T16:37:40,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060953) roll requested 2024-11-11T16:37:40,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060959 2024-11-11T16:37:40,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060959 2024-11-11T16:37:40,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060959) roll requested 2024-11-11T16:37:40,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060966 2024-11-11T16:37:40,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060966 2024-11-11T16:37:40,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060966) roll requested 2024-11-11T16:37:40,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060974 2024-11-11T16:37:40,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060974 2024-11-11T16:37:40,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060974) roll requested 2024-11-11T16:37:40,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060980 2024-11-11T16:37:40,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060980 2024-11-11T16:37:40,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060980) roll requested 2024-11-11T16:37:40,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060987 2024-11-11T16:37:40,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:40,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060987 2024-11-11T16:37:40,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:40,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:40,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060987) roll requested 2024-11-11T16:37:40,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343060997 2024-11-11T16:37:41,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060997 2024-11-11T16:37:41,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343060997) roll requested 2024-11-11T16:37:41,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061006 2024-11-11T16:37:41,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343060997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061006 2024-11-11T16:37:41,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061006) roll requested 2024-11-11T16:37:41,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061012 2024-11-11T16:37:41,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061012 2024-11-11T16:37:41,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061012) roll requested 2024-11-11T16:37:41,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061018 2024-11-11T16:37:41,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061018 2024-11-11T16:37:41,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061018) roll requested 2024-11-11T16:37:41,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061025 2024-11-11T16:37:41,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061025 2024-11-11T16:37:41,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061025) roll requested 2024-11-11T16:37:41,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061036 2024-11-11T16:37:41,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061036 2024-11-11T16:37:41,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061036) roll requested 2024-11-11T16:37:41,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061043 2024-11-11T16:37:41,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061043 2024-11-11T16:37:41,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061043) roll requested 2024-11-11T16:37:41,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061050 2024-11-11T16:37:41,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061050 2024-11-11T16:37:41,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061050) roll requested 2024-11-11T16:37:41,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061057 2024-11-11T16:37:41,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061057 2024-11-11T16:37:41,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061057) roll requested 2024-11-11T16:37:41,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061065 2024-11-11T16:37:41,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061065 2024-11-11T16:37:41,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061065) roll requested 2024-11-11T16:37:41,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061077 2024-11-11T16:37:41,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061077 2024-11-11T16:37:41,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061077) roll requested 2024-11-11T16:37:41,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061097 2024-11-11T16:37:41,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061097 2024-11-11T16:37:41,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061097) roll requested 2024-11-11T16:37:41,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061105 2024-11-11T16:37:41,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061105 2024-11-11T16:37:41,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061105) roll requested 2024-11-11T16:37:41,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061112 2024-11-11T16:37:41,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061112 2024-11-11T16:37:41,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061112) roll requested 2024-11-11T16:37:41,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061118 2024-11-11T16:37:41,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061118 2024-11-11T16:37:41,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061118) roll requested 2024-11-11T16:37:41,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061125 2024-11-11T16:37:41,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061125 2024-11-11T16:37:41,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061125) roll requested 2024-11-11T16:37:41,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061131 2024-11-11T16:37:41,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061131 2024-11-11T16:37:41,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061131) roll requested 2024-11-11T16:37:41,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061138 2024-11-11T16:37:41,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061138 2024-11-11T16:37:41,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061138) roll requested 2024-11-11T16:37:41,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061145 2024-11-11T16:37:41,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061145 2024-11-11T16:37:41,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061145) roll requested 2024-11-11T16:37:41,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061151 2024-11-11T16:37:41,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061151 2024-11-11T16:37:41,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061151) roll requested 2024-11-11T16:37:41,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061158 2024-11-11T16:37:41,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061158 2024-11-11T16:37:41,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061158) roll requested 2024-11-11T16:37:41,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061165 2024-11-11T16:37:41,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061165 2024-11-11T16:37:41,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061165) roll requested 2024-11-11T16:37:41,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061172 2024-11-11T16:37:41,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061172 2024-11-11T16:37:41,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061172) roll requested 2024-11-11T16:37:41,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061178 2024-11-11T16:37:41,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061178 2024-11-11T16:37:41,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061178) roll requested 2024-11-11T16:37:41,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061185 2024-11-11T16:37:41,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061185 2024-11-11T16:37:41,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061185) roll requested 2024-11-11T16:37:41,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061192 2024-11-11T16:37:41,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061192 2024-11-11T16:37:41,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061192) roll requested 2024-11-11T16:37:41,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061199 2024-11-11T16:37:41,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061199 2024-11-11T16:37:41,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061199) roll requested 2024-11-11T16:37:41,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061206 2024-11-11T16:37:41,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061206 2024-11-11T16:37:41,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061206) roll requested 2024-11-11T16:37:41,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061213 2024-11-11T16:37:41,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061213 2024-11-11T16:37:41,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061213) roll requested 2024-11-11T16:37:41,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061223 2024-11-11T16:37:41,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061223 2024-11-11T16:37:41,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061223) roll requested 2024-11-11T16:37:41,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061230 2024-11-11T16:37:41,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061230 2024-11-11T16:37:41,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061230) roll requested 2024-11-11T16:37:41,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061246 2024-11-11T16:37:41,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061246 2024-11-11T16:37:41,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061246) roll requested 2024-11-11T16:37:41,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061266 2024-11-11T16:37:41,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061266 2024-11-11T16:37:41,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061266) roll requested 2024-11-11T16:37:41,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061283 2024-11-11T16:37:41,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061283 2024-11-11T16:37:41,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061283) roll requested 2024-11-11T16:37:41,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061292 2024-11-11T16:37:41,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061292 2024-11-11T16:37:41,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061292) roll requested 2024-11-11T16:37:41,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061319 2024-11-11T16:37:41,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061319 2024-11-11T16:37:41,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061319) roll requested 2024-11-11T16:37:41,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061329 2024-11-11T16:37:41,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061329 2024-11-11T16:37:41,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061329) roll requested 2024-11-11T16:37:41,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061337 2024-11-11T16:37:41,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061337 2024-11-11T16:37:41,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061337) roll requested 2024-11-11T16:37:41,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061346 2024-11-11T16:37:41,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061346 2024-11-11T16:37:41,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061346) roll requested 2024-11-11T16:37:41,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061354 2024-11-11T16:37:41,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061354 2024-11-11T16:37:41,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061354) roll requested 2024-11-11T16:37:41,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061364 2024-11-11T16:37:41,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061364 2024-11-11T16:37:41,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061364) roll requested 2024-11-11T16:37:41,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061387 2024-11-11T16:37:41,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061387 2024-11-11T16:37:41,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061387) roll requested 2024-11-11T16:37:41,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061399 2024-11-11T16:37:41,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061399 2024-11-11T16:37:41,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061399) roll requested 2024-11-11T16:37:41,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061406 2024-11-11T16:37:41,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061406 2024-11-11T16:37:41,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061406) roll requested 2024-11-11T16:37:41,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061413 2024-11-11T16:37:41,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061413 2024-11-11T16:37:41,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061413) roll requested 2024-11-11T16:37:41,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061419 2024-11-11T16:37:41,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061419 2024-11-11T16:37:41,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061419) roll requested 2024-11-11T16:37:41,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061426 2024-11-11T16:37:41,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061426 2024-11-11T16:37:41,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061426) roll requested 2024-11-11T16:37:41,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061433 2024-11-11T16:37:41,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061433 2024-11-11T16:37:41,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061433) roll requested 2024-11-11T16:37:41,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061439 2024-11-11T16:37:41,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061439 2024-11-11T16:37:41,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061439) roll requested 2024-11-11T16:37:41,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061446 2024-11-11T16:37:41,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061446 2024-11-11T16:37:41,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061446) roll requested 2024-11-11T16:37:41,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061453 2024-11-11T16:37:41,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061453 2024-11-11T16:37:41,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061453) roll requested 2024-11-11T16:37:41,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061459 2024-11-11T16:37:41,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061459 2024-11-11T16:37:41,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061459) roll requested 2024-11-11T16:37:41,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061465 2024-11-11T16:37:41,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061465 2024-11-11T16:37:41,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061465) roll requested 2024-11-11T16:37:41,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061472 2024-11-11T16:37:41,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061472 2024-11-11T16:37:41,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061472) roll requested 2024-11-11T16:37:41,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061478 2024-11-11T16:37:41,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061478 2024-11-11T16:37:41,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061478) roll requested 2024-11-11T16:37:41,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061484 2024-11-11T16:37:41,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061484 2024-11-11T16:37:41,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061484) roll requested 2024-11-11T16:37:41,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061491 2024-11-11T16:37:41,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061491 2024-11-11T16:37:41,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061491) roll requested 2024-11-11T16:37:41,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061498 2024-11-11T16:37:41,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061498 2024-11-11T16:37:41,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061498) roll requested 2024-11-11T16:37:41,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061504 2024-11-11T16:37:41,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061504 2024-11-11T16:37:41,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061504) roll requested 2024-11-11T16:37:41,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061511 2024-11-11T16:37:41,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061511 2024-11-11T16:37:41,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061511) roll requested 2024-11-11T16:37:41,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061517 2024-11-11T16:37:41,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061517 2024-11-11T16:37:41,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061517) roll requested 2024-11-11T16:37:41,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061524 2024-11-11T16:37:41,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061524 2024-11-11T16:37:41,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061524) roll requested 2024-11-11T16:37:41,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061534 2024-11-11T16:37:41,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061534 2024-11-11T16:37:41,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061534) roll requested 2024-11-11T16:37:41,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061546 2024-11-11T16:37:41,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061546 2024-11-11T16:37:41,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061546) roll requested 2024-11-11T16:37:41,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061559 2024-11-11T16:37:41,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061559 2024-11-11T16:37:41,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061559) roll requested 2024-11-11T16:37:41,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061567 2024-11-11T16:37:41,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061567 2024-11-11T16:37:41,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061567) roll requested 2024-11-11T16:37:41,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061586 2024-11-11T16:37:41,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061586 2024-11-11T16:37:41,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061586) roll requested 2024-11-11T16:37:41,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061597 2024-11-11T16:37:41,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061597 2024-11-11T16:37:41,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061597) roll requested 2024-11-11T16:37:41,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061612 2024-11-11T16:37:41,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061612 2024-11-11T16:37:41,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061612) roll requested 2024-11-11T16:37:41,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061625 2024-11-11T16:37:41,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061625 2024-11-11T16:37:41,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061625) roll requested 2024-11-11T16:37:41,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061634 2024-11-11T16:37:41,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061634 2024-11-11T16:37:41,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061634) roll requested 2024-11-11T16:37:41,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061645 2024-11-11T16:37:41,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061645 2024-11-11T16:37:41,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061645) roll requested 2024-11-11T16:37:41,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061657 2024-11-11T16:37:41,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061657 2024-11-11T16:37:41,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061657) roll requested 2024-11-11T16:37:41,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061673 2024-11-11T16:37:41,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061673 2024-11-11T16:37:41,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061673) roll requested 2024-11-11T16:37:41,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061697 2024-11-11T16:37:41,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061697 2024-11-11T16:37:41,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061697) roll requested 2024-11-11T16:37:41,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061738 2024-11-11T16:37:41,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061738 2024-11-11T16:37:41,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061738) roll requested 2024-11-11T16:37:41,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061747 2024-11-11T16:37:41,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061747 2024-11-11T16:37:41,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061747) roll requested 2024-11-11T16:37:41,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061755 2024-11-11T16:37:41,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061755 2024-11-11T16:37:41,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061755) roll requested 2024-11-11T16:37:41,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061763 2024-11-11T16:37:41,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061763 2024-11-11T16:37:41,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061763) roll requested 2024-11-11T16:37:41,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061778 2024-11-11T16:37:41,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061778 2024-11-11T16:37:41,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061778) roll requested 2024-11-11T16:37:41,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061785 2024-11-11T16:37:41,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061785 2024-11-11T16:37:41,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061785) roll requested 2024-11-11T16:37:41,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061797 2024-11-11T16:37:41,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061797 2024-11-11T16:37:41,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061797) roll requested 2024-11-11T16:37:41,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061811 2024-11-11T16:37:41,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061811 2024-11-11T16:37:41,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061811) roll requested 2024-11-11T16:37:41,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061821 2024-11-11T16:37:41,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061821 2024-11-11T16:37:41,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061821) roll requested 2024-11-11T16:37:41,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061843 2024-11-11T16:37:41,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061843 2024-11-11T16:37:41,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061843) roll requested 2024-11-11T16:37:41,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061852 2024-11-11T16:37:41,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061852 2024-11-11T16:37:41,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061852) roll requested 2024-11-11T16:37:41,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061863 2024-11-11T16:37:41,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061863 2024-11-11T16:37:41,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061863) roll requested 2024-11-11T16:37:41,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061872 2024-11-11T16:37:41,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061872 2024-11-11T16:37:41,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061872) roll requested 2024-11-11T16:37:41,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061879 2024-11-11T16:37:41,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061879 2024-11-11T16:37:41,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061879) roll requested 2024-11-11T16:37:41,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061886 2024-11-11T16:37:41,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061886 2024-11-11T16:37:41,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061886) roll requested 2024-11-11T16:37:41,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061894 2024-11-11T16:37:41,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061894 2024-11-11T16:37:41,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061894) roll requested 2024-11-11T16:37:41,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061901 2024-11-11T16:37:41,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061901 2024-11-11T16:37:41,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061901) roll requested 2024-11-11T16:37:41,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061909 2024-11-11T16:37:41,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061909 2024-11-11T16:37:41,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061909) roll requested 2024-11-11T16:37:41,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061916 2024-11-11T16:37:41,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061916 2024-11-11T16:37:41,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061916) roll requested 2024-11-11T16:37:41,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061923 2024-11-11T16:37:41,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061923 2024-11-11T16:37:41,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061923) roll requested 2024-11-11T16:37:41,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061929 2024-11-11T16:37:41,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061929 2024-11-11T16:37:41,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061929) roll requested 2024-11-11T16:37:41,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061937 2024-11-11T16:37:41,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061937 2024-11-11T16:37:41,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061937) roll requested 2024-11-11T16:37:41,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061950 2024-11-11T16:37:41,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061950 2024-11-11T16:37:41,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061950) roll requested 2024-11-11T16:37:41,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061958 2024-11-11T16:37:41,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061958 2024-11-11T16:37:41,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061958) roll requested 2024-11-11T16:37:41,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061966 2024-11-11T16:37:41,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061966 2024-11-11T16:37:41,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061966) roll requested 2024-11-11T16:37:41,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061973 2024-11-11T16:37:41,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061973 2024-11-11T16:37:41,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061973) roll requested 2024-11-11T16:37:41,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061980 2024-11-11T16:37:41,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061980 2024-11-11T16:37:41,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061980) roll requested 2024-11-11T16:37:41,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061987 2024-11-11T16:37:41,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:41,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061987 2024-11-11T16:37:41,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:41,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:41,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061987) roll requested 2024-11-11T16:37:41,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343061994 2024-11-11T16:37:42,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061994 2024-11-11T16:37:42,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343061994) roll requested 2024-11-11T16:37:42,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062002 2024-11-11T16:37:42,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343061994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062002 2024-11-11T16:37:42,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062002) roll requested 2024-11-11T16:37:42,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062009 2024-11-11T16:37:42,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062009 2024-11-11T16:37:42,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062009) roll requested 2024-11-11T16:37:42,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062017 2024-11-11T16:37:42,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062017 2024-11-11T16:37:42,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062017) roll requested 2024-11-11T16:37:42,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062024 2024-11-11T16:37:42,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062024 2024-11-11T16:37:42,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062024) roll requested 2024-11-11T16:37:42,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062031 2024-11-11T16:37:42,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062031 2024-11-11T16:37:42,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062031) roll requested 2024-11-11T16:37:42,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062038 2024-11-11T16:37:42,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062038 2024-11-11T16:37:42,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062038) roll requested 2024-11-11T16:37:42,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062044 2024-11-11T16:37:42,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062044 2024-11-11T16:37:42,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062044) roll requested 2024-11-11T16:37:42,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062052 2024-11-11T16:37:42,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062052 2024-11-11T16:37:42,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062052) roll requested 2024-11-11T16:37:42,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062061 2024-11-11T16:37:42,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062061 2024-11-11T16:37:42,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062061) roll requested 2024-11-11T16:37:42,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062068 2024-11-11T16:37:42,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062068 2024-11-11T16:37:42,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062068) roll requested 2024-11-11T16:37:42,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062076 2024-11-11T16:37:42,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062076 2024-11-11T16:37:42,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062076) roll requested 2024-11-11T16:37:42,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062083 2024-11-11T16:37:42,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062083 2024-11-11T16:37:42,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062083) roll requested 2024-11-11T16:37:42,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062090 2024-11-11T16:37:42,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062090 2024-11-11T16:37:42,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062090) roll requested 2024-11-11T16:37:42,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062098 2024-11-11T16:37:42,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062098 2024-11-11T16:37:42,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062098) roll requested 2024-11-11T16:37:42,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062105 2024-11-11T16:37:42,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062105 2024-11-11T16:37:42,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062105) roll requested 2024-11-11T16:37:42,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062112 2024-11-11T16:37:42,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062112 2024-11-11T16:37:42,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062112) roll requested 2024-11-11T16:37:42,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062119 2024-11-11T16:37:42,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062119 2024-11-11T16:37:42,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062119) roll requested 2024-11-11T16:37:42,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062126 2024-11-11T16:37:42,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062126 2024-11-11T16:37:42,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062126) roll requested 2024-11-11T16:37:42,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062133 2024-11-11T16:37:42,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062133 2024-11-11T16:37:42,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062133) roll requested 2024-11-11T16:37:42,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062140 2024-11-11T16:37:42,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062140 2024-11-11T16:37:42,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062140) roll requested 2024-11-11T16:37:42,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062147 2024-11-11T16:37:42,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062147 2024-11-11T16:37:42,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062147) roll requested 2024-11-11T16:37:42,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062154 2024-11-11T16:37:42,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062154 2024-11-11T16:37:42,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062154) roll requested 2024-11-11T16:37:42,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062161 2024-11-11T16:37:42,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062161 2024-11-11T16:37:42,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062161) roll requested 2024-11-11T16:37:42,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062168 2024-11-11T16:37:42,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062168 2024-11-11T16:37:42,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062168) roll requested 2024-11-11T16:37:42,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062176 2024-11-11T16:37:42,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062176 2024-11-11T16:37:42,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062176) roll requested 2024-11-11T16:37:42,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062182 2024-11-11T16:37:42,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062182 2024-11-11T16:37:42,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062182) roll requested 2024-11-11T16:37:42,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062189 2024-11-11T16:37:42,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062189 2024-11-11T16:37:42,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062189) roll requested 2024-11-11T16:37:42,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062196 2024-11-11T16:37:42,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062196 2024-11-11T16:37:42,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062196) roll requested 2024-11-11T16:37:42,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062203 2024-11-11T16:37:42,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062203 2024-11-11T16:37:42,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062203) roll requested 2024-11-11T16:37:42,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062209 2024-11-11T16:37:42,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062209 2024-11-11T16:37:42,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062209) roll requested 2024-11-11T16:37:42,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062216 2024-11-11T16:37:42,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062216 2024-11-11T16:37:42,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062216) roll requested 2024-11-11T16:37:42,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062223 2024-11-11T16:37:42,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062223 2024-11-11T16:37:42,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062223) roll requested 2024-11-11T16:37:42,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062230 2024-11-11T16:37:42,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062230 2024-11-11T16:37:42,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062230) roll requested 2024-11-11T16:37:42,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062237 2024-11-11T16:37:42,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062237 2024-11-11T16:37:42,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062237) roll requested 2024-11-11T16:37:42,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062244 2024-11-11T16:37:42,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062244 2024-11-11T16:37:42,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062244) roll requested 2024-11-11T16:37:42,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062251 2024-11-11T16:37:42,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062251 2024-11-11T16:37:42,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062251) roll requested 2024-11-11T16:37:42,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062259 2024-11-11T16:37:42,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062259 2024-11-11T16:37:42,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062259) roll requested 2024-11-11T16:37:42,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062266 2024-11-11T16:37:42,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062266 2024-11-11T16:37:42,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062266) roll requested 2024-11-11T16:37:42,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062273 2024-11-11T16:37:42,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062273 2024-11-11T16:37:42,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062273) roll requested 2024-11-11T16:37:42,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062280 2024-11-11T16:37:42,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062280 2024-11-11T16:37:42,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062280) roll requested 2024-11-11T16:37:42,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062287 2024-11-11T16:37:42,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062287 2024-11-11T16:37:42,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062287) roll requested 2024-11-11T16:37:42,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062295 2024-11-11T16:37:42,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062295 2024-11-11T16:37:42,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062295) roll requested 2024-11-11T16:37:42,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062302 2024-11-11T16:37:42,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062302 2024-11-11T16:37:42,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062302) roll requested 2024-11-11T16:37:42,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062309 2024-11-11T16:37:42,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062309 2024-11-11T16:37:42,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062309) roll requested 2024-11-11T16:37:42,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062316 2024-11-11T16:37:42,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062316 2024-11-11T16:37:42,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062316) roll requested 2024-11-11T16:37:42,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062323 2024-11-11T16:37:42,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062323 2024-11-11T16:37:42,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062323) roll requested 2024-11-11T16:37:42,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062331 2024-11-11T16:37:42,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062331 2024-11-11T16:37:42,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062331) roll requested 2024-11-11T16:37:42,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062347 2024-11-11T16:37:42,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062347 2024-11-11T16:37:42,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062347) roll requested 2024-11-11T16:37:42,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062361 2024-11-11T16:37:42,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062361 2024-11-11T16:37:42,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062361) roll requested 2024-11-11T16:37:42,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062368 2024-11-11T16:37:42,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062368 2024-11-11T16:37:42,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062368) roll requested 2024-11-11T16:37:42,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062376 2024-11-11T16:37:42,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062376 2024-11-11T16:37:42,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062376) roll requested 2024-11-11T16:37:42,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062384 2024-11-11T16:37:42,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062384 2024-11-11T16:37:42,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062384) roll requested 2024-11-11T16:37:42,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062391 2024-11-11T16:37:42,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062391 2024-11-11T16:37:42,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062391) roll requested 2024-11-11T16:37:42,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062399 2024-11-11T16:37:42,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062399 2024-11-11T16:37:42,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062399) roll requested 2024-11-11T16:37:42,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062406 2024-11-11T16:37:42,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062406 2024-11-11T16:37:42,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062406) roll requested 2024-11-11T16:37:42,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062413 2024-11-11T16:37:42,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062413 2024-11-11T16:37:42,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062413) roll requested 2024-11-11T16:37:42,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062422 2024-11-11T16:37:42,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062422 2024-11-11T16:37:42,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062422) roll requested 2024-11-11T16:37:42,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062430 2024-11-11T16:37:42,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062430 2024-11-11T16:37:42,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062430) roll requested 2024-11-11T16:37:42,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062439 2024-11-11T16:37:42,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062439 2024-11-11T16:37:42,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062439) roll requested 2024-11-11T16:37:42,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062450 2024-11-11T16:37:42,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062450 2024-11-11T16:37:42,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062450) roll requested 2024-11-11T16:37:42,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062457 2024-11-11T16:37:42,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062457 2024-11-11T16:37:42,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062457) roll requested 2024-11-11T16:37:42,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062465 2024-11-11T16:37:42,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062465 2024-11-11T16:37:42,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062465) roll requested 2024-11-11T16:37:42,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062472 2024-11-11T16:37:42,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062472 2024-11-11T16:37:42,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062472) roll requested 2024-11-11T16:37:42,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062481 2024-11-11T16:37:42,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062481 2024-11-11T16:37:42,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062481) roll requested 2024-11-11T16:37:42,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062492 2024-11-11T16:37:42,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062492 2024-11-11T16:37:42,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062492) roll requested 2024-11-11T16:37:42,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062503 2024-11-11T16:37:42,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062503 2024-11-11T16:37:42,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062503) roll requested 2024-11-11T16:37:42,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062513 2024-11-11T16:37:42,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062513 2024-11-11T16:37:42,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062513) roll requested 2024-11-11T16:37:42,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062520 2024-11-11T16:37:42,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062520 2024-11-11T16:37:42,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062520) roll requested 2024-11-11T16:37:42,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062527 2024-11-11T16:37:42,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062527 2024-11-11T16:37:42,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062527) roll requested 2024-11-11T16:37:42,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062534 2024-11-11T16:37:42,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062534 2024-11-11T16:37:42,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062534) roll requested 2024-11-11T16:37:42,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062541 2024-11-11T16:37:42,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062541 2024-11-11T16:37:42,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062541) roll requested 2024-11-11T16:37:42,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062547 2024-11-11T16:37:42,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062547 2024-11-11T16:37:42,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062547) roll requested 2024-11-11T16:37:42,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062555 2024-11-11T16:37:42,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062555 2024-11-11T16:37:42,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062555) roll requested 2024-11-11T16:37:42,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062565 2024-11-11T16:37:42,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062565 2024-11-11T16:37:42,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062565) roll requested 2024-11-11T16:37:42,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062573 2024-11-11T16:37:42,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062573 2024-11-11T16:37:42,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062573) roll requested 2024-11-11T16:37:42,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062589 2024-11-11T16:37:42,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062589 2024-11-11T16:37:42,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062589) roll requested 2024-11-11T16:37:42,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062597 2024-11-11T16:37:42,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062597 2024-11-11T16:37:42,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062597) roll requested 2024-11-11T16:37:42,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062605 2024-11-11T16:37:42,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062605 2024-11-11T16:37:42,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062605) roll requested 2024-11-11T16:37:42,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062613 2024-11-11T16:37:42,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062613 2024-11-11T16:37:42,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062613) roll requested 2024-11-11T16:37:42,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062621 2024-11-11T16:37:42,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062621 2024-11-11T16:37:42,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062621) roll requested 2024-11-11T16:37:42,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062634 2024-11-11T16:37:42,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062634 2024-11-11T16:37:42,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062634) roll requested 2024-11-11T16:37:42,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062641 2024-11-11T16:37:42,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062641 2024-11-11T16:37:42,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062641) roll requested 2024-11-11T16:37:42,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062649 2024-11-11T16:37:42,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062649 2024-11-11T16:37:42,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062649) roll requested 2024-11-11T16:37:42,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062657 2024-11-11T16:37:42,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062657 2024-11-11T16:37:42,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062657) roll requested 2024-11-11T16:37:42,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062664 2024-11-11T16:37:42,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062664 2024-11-11T16:37:42,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062664) roll requested 2024-11-11T16:37:42,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062673 2024-11-11T16:37:42,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062673 2024-11-11T16:37:42,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062673) roll requested 2024-11-11T16:37:42,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062681 2024-11-11T16:37:42,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062681 2024-11-11T16:37:42,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062681) roll requested 2024-11-11T16:37:42,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062687 2024-11-11T16:37:42,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062687 2024-11-11T16:37:42,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062687) roll requested 2024-11-11T16:37:42,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062695 2024-11-11T16:37:42,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062695 2024-11-11T16:37:42,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062695) roll requested 2024-11-11T16:37:42,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062702 2024-11-11T16:37:42,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062702 2024-11-11T16:37:42,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062702) roll requested 2024-11-11T16:37:42,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062710 2024-11-11T16:37:42,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062710 2024-11-11T16:37:42,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062710) roll requested 2024-11-11T16:37:42,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062721 2024-11-11T16:37:42,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062721 2024-11-11T16:37:42,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062721) roll requested 2024-11-11T16:37:42,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062728 2024-11-11T16:37:42,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062728 2024-11-11T16:37:42,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062728) roll requested 2024-11-11T16:37:42,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062735 2024-11-11T16:37:42,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062735 2024-11-11T16:37:42,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062735) roll requested 2024-11-11T16:37:42,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062742 2024-11-11T16:37:42,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062742 2024-11-11T16:37:42,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062742) roll requested 2024-11-11T16:37:42,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062749 2024-11-11T16:37:42,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062749 2024-11-11T16:37:42,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062749) roll requested 2024-11-11T16:37:42,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062757 2024-11-11T16:37:42,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062757 2024-11-11T16:37:42,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062757) roll requested 2024-11-11T16:37:42,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062764 2024-11-11T16:37:42,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062764 2024-11-11T16:37:42,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062764) roll requested 2024-11-11T16:37:42,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062771 2024-11-11T16:37:42,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062771 2024-11-11T16:37:42,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062771) roll requested 2024-11-11T16:37:42,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062778 2024-11-11T16:37:42,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062778 2024-11-11T16:37:42,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062778) roll requested 2024-11-11T16:37:42,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062785 2024-11-11T16:37:42,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062785 2024-11-11T16:37:42,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062785) roll requested 2024-11-11T16:37:42,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062795 2024-11-11T16:37:42,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062795 2024-11-11T16:37:42,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062795) roll requested 2024-11-11T16:37:42,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062802 2024-11-11T16:37:42,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062802 2024-11-11T16:37:42,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062802) roll requested 2024-11-11T16:37:42,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062810 2024-11-11T16:37:42,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062810 2024-11-11T16:37:42,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062810) roll requested 2024-11-11T16:37:42,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062818 2024-11-11T16:37:42,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062818 2024-11-11T16:37:42,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062818) roll requested 2024-11-11T16:37:42,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062825 2024-11-11T16:37:42,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062825 2024-11-11T16:37:42,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062825) roll requested 2024-11-11T16:37:42,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062833 2024-11-11T16:37:42,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062833 2024-11-11T16:37:42,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062833) roll requested 2024-11-11T16:37:42,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062840 2024-11-11T16:37:42,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062840 2024-11-11T16:37:42,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062840) roll requested 2024-11-11T16:37:42,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062848 2024-11-11T16:37:42,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062848 2024-11-11T16:37:42,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062848) roll requested 2024-11-11T16:37:42,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062856 2024-11-11T16:37:42,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062856 2024-11-11T16:37:42,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062856) roll requested 2024-11-11T16:37:42,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062863 2024-11-11T16:37:42,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062863 2024-11-11T16:37:42,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062863) roll requested 2024-11-11T16:37:42,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062870 2024-11-11T16:37:42,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062870 2024-11-11T16:37:42,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062870) roll requested 2024-11-11T16:37:42,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062878 2024-11-11T16:37:42,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062878 2024-11-11T16:37:42,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062878) roll requested 2024-11-11T16:37:42,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062886 2024-11-11T16:37:42,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062886 2024-11-11T16:37:42,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062886) roll requested 2024-11-11T16:37:42,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062893 2024-11-11T16:37:42,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062893 2024-11-11T16:37:42,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062893) roll requested 2024-11-11T16:37:42,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062901 2024-11-11T16:37:42,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062901 2024-11-11T16:37:42,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062901) roll requested 2024-11-11T16:37:42,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062908 2024-11-11T16:37:42,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062908 2024-11-11T16:37:42,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062908) roll requested 2024-11-11T16:37:42,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062916 2024-11-11T16:37:42,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062916 2024-11-11T16:37:42,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062916) roll requested 2024-11-11T16:37:42,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062926 2024-11-11T16:37:42,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062926 2024-11-11T16:37:42,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062926) roll requested 2024-11-11T16:37:42,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062935 2024-11-11T16:37:42,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062935 2024-11-11T16:37:42,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062935) roll requested 2024-11-11T16:37:42,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062953 2024-11-11T16:37:42,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062953 2024-11-11T16:37:42,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062953) roll requested 2024-11-11T16:37:42,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062965 2024-11-11T16:37:42,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062965 2024-11-11T16:37:42,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062965) roll requested 2024-11-11T16:37:42,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062973 2024-11-11T16:37:42,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062973 2024-11-11T16:37:42,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062973) roll requested 2024-11-11T16:37:42,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062981 2024-11-11T16:37:42,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062981 2024-11-11T16:37:42,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062981) roll requested 2024-11-11T16:37:42,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062989 2024-11-11T16:37:42,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:42,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062989 2024-11-11T16:37:42,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:42,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:42,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062989) roll requested 2024-11-11T16:37:42,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343062997 2024-11-11T16:37:43,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062997 2024-11-11T16:37:43,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343062997) roll requested 2024-11-11T16:37:43,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063004 2024-11-11T16:37:43,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343062997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063004 2024-11-11T16:37:43,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063004) roll requested 2024-11-11T16:37:43,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063016 2024-11-11T16:37:43,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063016 2024-11-11T16:37:43,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063016) roll requested 2024-11-11T16:37:43,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063026 2024-11-11T16:37:43,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063026 2024-11-11T16:37:43,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063026) roll requested 2024-11-11T16:37:43,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063035 2024-11-11T16:37:43,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063035 2024-11-11T16:37:43,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063035) roll requested 2024-11-11T16:37:43,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063043 2024-11-11T16:37:43,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063043 2024-11-11T16:37:43,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063043) roll requested 2024-11-11T16:37:43,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063057 2024-11-11T16:37:43,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063057 2024-11-11T16:37:43,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063057) roll requested 2024-11-11T16:37:43,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063067 2024-11-11T16:37:43,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063067 2024-11-11T16:37:43,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063067) roll requested 2024-11-11T16:37:43,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063077 2024-11-11T16:37:43,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063077 2024-11-11T16:37:43,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063077) roll requested 2024-11-11T16:37:43,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063084 2024-11-11T16:37:43,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063084 2024-11-11T16:37:43,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063084) roll requested 2024-11-11T16:37:43,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063096 2024-11-11T16:37:43,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063096 2024-11-11T16:37:43,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063096) roll requested 2024-11-11T16:37:43,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063103 2024-11-11T16:37:43,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063103 2024-11-11T16:37:43,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063103) roll requested 2024-11-11T16:37:43,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063110 2024-11-11T16:37:43,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063110 2024-11-11T16:37:43,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063110) roll requested 2024-11-11T16:37:43,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063117 2024-11-11T16:37:43,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063117 2024-11-11T16:37:43,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063117) roll requested 2024-11-11T16:37:43,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063125 2024-11-11T16:37:43,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063125 2024-11-11T16:37:43,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063125) roll requested 2024-11-11T16:37:43,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063134 2024-11-11T16:37:43,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063134 2024-11-11T16:37:43,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063134) roll requested 2024-11-11T16:37:43,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063141 2024-11-11T16:37:43,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063141 2024-11-11T16:37:43,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063141) roll requested 2024-11-11T16:37:43,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063148 2024-11-11T16:37:43,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063148 2024-11-11T16:37:43,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063148) roll requested 2024-11-11T16:37:43,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063156 2024-11-11T16:37:43,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063156 2024-11-11T16:37:43,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063156) roll requested 2024-11-11T16:37:43,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063163 2024-11-11T16:37:43,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063163 2024-11-11T16:37:43,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063163) roll requested 2024-11-11T16:37:43,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063170 2024-11-11T16:37:43,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063170 2024-11-11T16:37:43,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063170) roll requested 2024-11-11T16:37:43,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063177 2024-11-11T16:37:43,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063177 2024-11-11T16:37:43,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063177) roll requested 2024-11-11T16:37:43,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063184 2024-11-11T16:37:43,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063184 2024-11-11T16:37:43,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063184) roll requested 2024-11-11T16:37:43,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063192 2024-11-11T16:37:43,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063192 2024-11-11T16:37:43,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063192) roll requested 2024-11-11T16:37:43,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063201 2024-11-11T16:37:43,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063201 2024-11-11T16:37:43,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063201) roll requested 2024-11-11T16:37:43,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063208 2024-11-11T16:37:43,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063208 2024-11-11T16:37:43,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063208) roll requested 2024-11-11T16:37:43,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063215 2024-11-11T16:37:43,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063215 2024-11-11T16:37:43,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063215) roll requested 2024-11-11T16:37:43,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063222 2024-11-11T16:37:43,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063222 2024-11-11T16:37:43,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063222) roll requested 2024-11-11T16:37:43,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063229 2024-11-11T16:37:43,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063229 2024-11-11T16:37:43,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063229) roll requested 2024-11-11T16:37:43,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063238 2024-11-11T16:37:43,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063238 2024-11-11T16:37:43,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063238) roll requested 2024-11-11T16:37:43,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063245 2024-11-11T16:37:43,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063245 2024-11-11T16:37:43,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063245) roll requested 2024-11-11T16:37:43,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063252 2024-11-11T16:37:43,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063252 2024-11-11T16:37:43,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063252) roll requested 2024-11-11T16:37:43,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063259 2024-11-11T16:37:43,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063259 2024-11-11T16:37:43,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063259) roll requested 2024-11-11T16:37:43,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063265 2024-11-11T16:37:43,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063265 2024-11-11T16:37:43,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063265) roll requested 2024-11-11T16:37:43,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063273 2024-11-11T16:37:43,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063273 2024-11-11T16:37:43,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063273) roll requested 2024-11-11T16:37:43,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063280 2024-11-11T16:37:43,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063280 2024-11-11T16:37:43,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063280) roll requested 2024-11-11T16:37:43,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063287 2024-11-11T16:37:43,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063287 2024-11-11T16:37:43,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063287) roll requested 2024-11-11T16:37:43,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063295 2024-11-11T16:37:43,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063295 2024-11-11T16:37:43,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063295) roll requested 2024-11-11T16:37:43,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063302 2024-11-11T16:37:43,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063302 2024-11-11T16:37:43,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063302) roll requested 2024-11-11T16:37:43,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063313 2024-11-11T16:37:43,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063313 2024-11-11T16:37:43,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063313) roll requested 2024-11-11T16:37:43,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063321 2024-11-11T16:37:43,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063321 2024-11-11T16:37:43,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063321) roll requested 2024-11-11T16:37:43,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063328 2024-11-11T16:37:43,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063328 2024-11-11T16:37:43,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063328) roll requested 2024-11-11T16:37:43,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063336 2024-11-11T16:37:43,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063336 2024-11-11T16:37:43,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063336) roll requested 2024-11-11T16:37:43,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063342 2024-11-11T16:37:43,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063342 2024-11-11T16:37:43,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063342) roll requested 2024-11-11T16:37:43,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063353 2024-11-11T16:37:43,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063353 2024-11-11T16:37:43,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063353) roll requested 2024-11-11T16:37:43,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063363 2024-11-11T16:37:43,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063363 2024-11-11T16:37:43,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063363) roll requested 2024-11-11T16:37:43,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063370 2024-11-11T16:37:43,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063370 2024-11-11T16:37:43,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063370) roll requested 2024-11-11T16:37:43,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063378 2024-11-11T16:37:43,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063378 2024-11-11T16:37:43,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063378) roll requested 2024-11-11T16:37:43,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063386 2024-11-11T16:37:43,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063386 2024-11-11T16:37:43,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063386) roll requested 2024-11-11T16:37:43,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063394 2024-11-11T16:37:43,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063394 2024-11-11T16:37:43,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063394) roll requested 2024-11-11T16:37:43,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063401 2024-11-11T16:37:43,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063401 2024-11-11T16:37:43,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063401) roll requested 2024-11-11T16:37:43,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063411 2024-11-11T16:37:43,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063411 2024-11-11T16:37:43,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063411) roll requested 2024-11-11T16:37:43,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063420 2024-11-11T16:37:43,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063420 2024-11-11T16:37:43,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063420) roll requested 2024-11-11T16:37:43,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063427 2024-11-11T16:37:43,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063427 2024-11-11T16:37:43,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063427) roll requested 2024-11-11T16:37:43,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063434 2024-11-11T16:37:43,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063434 2024-11-11T16:37:43,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063434) roll requested 2024-11-11T16:37:43,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063441 2024-11-11T16:37:43,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063441 2024-11-11T16:37:43,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063441) roll requested 2024-11-11T16:37:43,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063449 2024-11-11T16:37:43,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063449 2024-11-11T16:37:43,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063449) roll requested 2024-11-11T16:37:43,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063458 2024-11-11T16:37:43,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063458 2024-11-11T16:37:43,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063458) roll requested 2024-11-11T16:37:43,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063486 2024-11-11T16:37:43,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063486 2024-11-11T16:37:43,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063486) roll requested 2024-11-11T16:37:43,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063501 2024-11-11T16:37:43,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063501 2024-11-11T16:37:43,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063501) roll requested 2024-11-11T16:37:43,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063509 2024-11-11T16:37:43,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063509 2024-11-11T16:37:43,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063509) roll requested 2024-11-11T16:37:43,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063516 2024-11-11T16:37:43,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063516 2024-11-11T16:37:43,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063516) roll requested 2024-11-11T16:37:43,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063524 2024-11-11T16:37:43,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063524 2024-11-11T16:37:43,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063524) roll requested 2024-11-11T16:37:43,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063531 2024-11-11T16:37:43,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063531 2024-11-11T16:37:43,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063531) roll requested 2024-11-11T16:37:43,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063539 2024-11-11T16:37:43,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063539 2024-11-11T16:37:43,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063539) roll requested 2024-11-11T16:37:43,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063546 2024-11-11T16:37:43,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063546 2024-11-11T16:37:43,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063546) roll requested 2024-11-11T16:37:43,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063553 2024-11-11T16:37:43,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063553 2024-11-11T16:37:43,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063553) roll requested 2024-11-11T16:37:43,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063560 2024-11-11T16:37:43,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063560 2024-11-11T16:37:43,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063560) roll requested 2024-11-11T16:37:43,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063567 2024-11-11T16:37:43,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063567 2024-11-11T16:37:43,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063567) roll requested 2024-11-11T16:37:43,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063573 2024-11-11T16:37:43,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063573 2024-11-11T16:37:43,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063573) roll requested 2024-11-11T16:37:43,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063581 2024-11-11T16:37:43,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063581 2024-11-11T16:37:43,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063581) roll requested 2024-11-11T16:37:43,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063592 2024-11-11T16:37:43,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063592 2024-11-11T16:37:43,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063592) roll requested 2024-11-11T16:37:43,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063599 2024-11-11T16:37:43,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063599 2024-11-11T16:37:43,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063599) roll requested 2024-11-11T16:37:43,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063605 2024-11-11T16:37:43,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063605 2024-11-11T16:37:43,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063605) roll requested 2024-11-11T16:37:43,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063611 2024-11-11T16:37:43,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063611 2024-11-11T16:37:43,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063611) roll requested 2024-11-11T16:37:43,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063618 2024-11-11T16:37:43,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063618 2024-11-11T16:37:43,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063618) roll requested 2024-11-11T16:37:43,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063624 2024-11-11T16:37:43,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063624 2024-11-11T16:37:43,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063624) roll requested 2024-11-11T16:37:43,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063630 2024-11-11T16:37:43,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063630 2024-11-11T16:37:43,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063630) roll requested 2024-11-11T16:37:43,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063637 2024-11-11T16:37:43,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063637 2024-11-11T16:37:43,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063637) roll requested 2024-11-11T16:37:43,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063643 2024-11-11T16:37:43,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063643 2024-11-11T16:37:43,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063643) roll requested 2024-11-11T16:37:43,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063649 2024-11-11T16:37:43,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063649 2024-11-11T16:37:43,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063649) roll requested 2024-11-11T16:37:43,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063656 2024-11-11T16:37:43,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063656 2024-11-11T16:37:43,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063656) roll requested 2024-11-11T16:37:43,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063662 2024-11-11T16:37:43,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063662 2024-11-11T16:37:43,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063662) roll requested 2024-11-11T16:37:43,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063668 2024-11-11T16:37:43,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063668 2024-11-11T16:37:43,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063668) roll requested 2024-11-11T16:37:43,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063675 2024-11-11T16:37:43,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063675 2024-11-11T16:37:43,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063675) roll requested 2024-11-11T16:37:43,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063681 2024-11-11T16:37:43,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063681 2024-11-11T16:37:43,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063681) roll requested 2024-11-11T16:37:43,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063688 2024-11-11T16:37:43,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063688 2024-11-11T16:37:43,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063688) roll requested 2024-11-11T16:37:43,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063694 2024-11-11T16:37:43,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063694 2024-11-11T16:37:43,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063694) roll requested 2024-11-11T16:37:43,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063700 2024-11-11T16:37:43,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063700 2024-11-11T16:37:43,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063700) roll requested 2024-11-11T16:37:43,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063707 2024-11-11T16:37:43,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063707 2024-11-11T16:37:43,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063707) roll requested 2024-11-11T16:37:43,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063714 2024-11-11T16:37:43,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063714 2024-11-11T16:37:43,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063714) roll requested 2024-11-11T16:37:43,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063720 2024-11-11T16:37:43,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063720 2024-11-11T16:37:43,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063720) roll requested 2024-11-11T16:37:43,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063726 2024-11-11T16:37:43,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063726 2024-11-11T16:37:43,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063726) roll requested 2024-11-11T16:37:43,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063733 2024-11-11T16:37:43,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063733 2024-11-11T16:37:43,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063733) roll requested 2024-11-11T16:37:43,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063739 2024-11-11T16:37:43,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063739 2024-11-11T16:37:43,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063739) roll requested 2024-11-11T16:37:43,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063746 2024-11-11T16:37:43,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063746 2024-11-11T16:37:43,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063746) roll requested 2024-11-11T16:37:43,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063758 2024-11-11T16:37:43,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063758 2024-11-11T16:37:43,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063758) roll requested 2024-11-11T16:37:43,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063769 2024-11-11T16:37:43,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063769 2024-11-11T16:37:43,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063769) roll requested 2024-11-11T16:37:43,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063781 2024-11-11T16:37:43,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063781 2024-11-11T16:37:43,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063781) roll requested 2024-11-11T16:37:43,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063789 2024-11-11T16:37:43,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063789 2024-11-11T16:37:43,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063789) roll requested 2024-11-11T16:37:43,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063797 2024-11-11T16:37:43,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063797 2024-11-11T16:37:43,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063797) roll requested 2024-11-11T16:37:43,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063804 2024-11-11T16:37:43,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063804 2024-11-11T16:37:43,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063804) roll requested 2024-11-11T16:37:43,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063813 2024-11-11T16:37:43,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063813 2024-11-11T16:37:43,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063813) roll requested 2024-11-11T16:37:43,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063820 2024-11-11T16:37:43,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063820 2024-11-11T16:37:43,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063820) roll requested 2024-11-11T16:37:43,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063829 2024-11-11T16:37:43,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063829 2024-11-11T16:37:43,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063829) roll requested 2024-11-11T16:37:43,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063837 2024-11-11T16:37:43,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063837 2024-11-11T16:37:43,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063837) roll requested 2024-11-11T16:37:43,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063844 2024-11-11T16:37:43,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063844 2024-11-11T16:37:43,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063844) roll requested 2024-11-11T16:37:43,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063851 2024-11-11T16:37:43,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063851 2024-11-11T16:37:43,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063851) roll requested 2024-11-11T16:37:43,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063857 2024-11-11T16:37:43,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063857 2024-11-11T16:37:43,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063857) roll requested 2024-11-11T16:37:43,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063868 2024-11-11T16:37:43,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063868 2024-11-11T16:37:43,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063868) roll requested 2024-11-11T16:37:43,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063876 2024-11-11T16:37:43,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063876 2024-11-11T16:37:43,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063876) roll requested 2024-11-11T16:37:43,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063883 2024-11-11T16:37:43,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063883 2024-11-11T16:37:43,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063883) roll requested 2024-11-11T16:37:43,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063891 2024-11-11T16:37:43,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063891 2024-11-11T16:37:43,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063891) roll requested 2024-11-11T16:37:43,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063899 2024-11-11T16:37:43,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063899 2024-11-11T16:37:43,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063899) roll requested 2024-11-11T16:37:43,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063917 2024-11-11T16:37:43,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063917 2024-11-11T16:37:43,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063917) roll requested 2024-11-11T16:37:43,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063928 2024-11-11T16:37:43,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063928 2024-11-11T16:37:43,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063928) roll requested 2024-11-11T16:37:43,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063938 2024-11-11T16:37:43,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063938 2024-11-11T16:37:43,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063938) roll requested 2024-11-11T16:37:43,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063946 2024-11-11T16:37:43,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063946 2024-11-11T16:37:43,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063946) roll requested 2024-11-11T16:37:43,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063953 2024-11-11T16:37:43,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063953 2024-11-11T16:37:43,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063953) roll requested 2024-11-11T16:37:43,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063959 2024-11-11T16:37:43,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063959 2024-11-11T16:37:43,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063959) roll requested 2024-11-11T16:37:43,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063966 2024-11-11T16:37:43,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063966 2024-11-11T16:37:43,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063966) roll requested 2024-11-11T16:37:43,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063972 2024-11-11T16:37:43,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063972 2024-11-11T16:37:43,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063972) roll requested 2024-11-11T16:37:43,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063978 2024-11-11T16:37:43,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063978 2024-11-11T16:37:43,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063978) roll requested 2024-11-11T16:37:43,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063985 2024-11-11T16:37:43,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063985 2024-11-11T16:37:43,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063985) roll requested 2024-11-11T16:37:43,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063991 2024-11-11T16:37:43,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:43,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063991 2024-11-11T16:37:43,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:43,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:43,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063991) roll requested 2024-11-11T16:37:43,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343063998 2024-11-11T16:37:44,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063998 2024-11-11T16:37:44,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343063998) roll requested 2024-11-11T16:37:44,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064004 2024-11-11T16:37:44,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343063998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064004 2024-11-11T16:37:44,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064004) roll requested 2024-11-11T16:37:44,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064011 2024-11-11T16:37:44,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064011 2024-11-11T16:37:44,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064011) roll requested 2024-11-11T16:37:44,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064017 2024-11-11T16:37:44,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064017 2024-11-11T16:37:44,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064017) roll requested 2024-11-11T16:37:44,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064023 2024-11-11T16:37:44,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064023 2024-11-11T16:37:44,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064023) roll requested 2024-11-11T16:37:44,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064030 2024-11-11T16:37:44,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064030 2024-11-11T16:37:44,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064030) roll requested 2024-11-11T16:37:44,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064037 2024-11-11T16:37:44,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064037 2024-11-11T16:37:44,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064037) roll requested 2024-11-11T16:37:44,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064047 2024-11-11T16:37:44,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064047 2024-11-11T16:37:44,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064047) roll requested 2024-11-11T16:37:44,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064054 2024-11-11T16:37:44,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064054 2024-11-11T16:37:44,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064054) roll requested 2024-11-11T16:37:44,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064061 2024-11-11T16:37:44,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064061 2024-11-11T16:37:44,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064061) roll requested 2024-11-11T16:37:44,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064067 2024-11-11T16:37:44,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064067 2024-11-11T16:37:44,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064067) roll requested 2024-11-11T16:37:44,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064073 2024-11-11T16:37:44,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064073 2024-11-11T16:37:44,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064073) roll requested 2024-11-11T16:37:44,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064079 2024-11-11T16:37:44,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064079 2024-11-11T16:37:44,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064079) roll requested 2024-11-11T16:37:44,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064086 2024-11-11T16:37:44,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064086 2024-11-11T16:37:44,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064086) roll requested 2024-11-11T16:37:44,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064093 2024-11-11T16:37:44,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064093 2024-11-11T16:37:44,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064093) roll requested 2024-11-11T16:37:44,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064100 2024-11-11T16:37:44,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064100 2024-11-11T16:37:44,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064100) roll requested 2024-11-11T16:37:44,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064106 2024-11-11T16:37:44,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064106 2024-11-11T16:37:44,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064106) roll requested 2024-11-11T16:37:44,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064112 2024-11-11T16:37:44,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064112 2024-11-11T16:37:44,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064112) roll requested 2024-11-11T16:37:44,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064118 2024-11-11T16:37:44,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064118 2024-11-11T16:37:44,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064118) roll requested 2024-11-11T16:37:44,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064125 2024-11-11T16:37:44,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064125 2024-11-11T16:37:44,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064125) roll requested 2024-11-11T16:37:44,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064131 2024-11-11T16:37:44,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064131 2024-11-11T16:37:44,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064131) roll requested 2024-11-11T16:37:44,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064137 2024-11-11T16:37:44,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064137 2024-11-11T16:37:44,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064137) roll requested 2024-11-11T16:37:44,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064144 2024-11-11T16:37:44,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064144 2024-11-11T16:37:44,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064144) roll requested 2024-11-11T16:37:44,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064150 2024-11-11T16:37:44,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064150 2024-11-11T16:37:44,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064150) roll requested 2024-11-11T16:37:44,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064157 2024-11-11T16:37:44,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064157 2024-11-11T16:37:44,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064157) roll requested 2024-11-11T16:37:44,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064163 2024-11-11T16:37:44,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064163 2024-11-11T16:37:44,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064163) roll requested 2024-11-11T16:37:44,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064169 2024-11-11T16:37:44,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064169 2024-11-11T16:37:44,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064169) roll requested 2024-11-11T16:37:44,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064175 2024-11-11T16:37:44,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064175 2024-11-11T16:37:44,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064175) roll requested 2024-11-11T16:37:44,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064182 2024-11-11T16:37:44,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064182 2024-11-11T16:37:44,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064182) roll requested 2024-11-11T16:37:44,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064189 2024-11-11T16:37:44,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064189 2024-11-11T16:37:44,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064189) roll requested 2024-11-11T16:37:44,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064200 2024-11-11T16:37:44,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064200 2024-11-11T16:37:44,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064200) roll requested 2024-11-11T16:37:44,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064209 2024-11-11T16:37:44,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064209 2024-11-11T16:37:44,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064209) roll requested 2024-11-11T16:37:44,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064220 2024-11-11T16:37:44,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064220 2024-11-11T16:37:44,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064220) roll requested 2024-11-11T16:37:44,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064233 2024-11-11T16:37:44,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064233 2024-11-11T16:37:44,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064233) roll requested 2024-11-11T16:37:44,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064244 2024-11-11T16:37:44,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064244 2024-11-11T16:37:44,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064244) roll requested 2024-11-11T16:37:44,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064254 2024-11-11T16:37:44,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064254 2024-11-11T16:37:44,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064254) roll requested 2024-11-11T16:37:44,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064264 2024-11-11T16:37:44,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064264 2024-11-11T16:37:44,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064264) roll requested 2024-11-11T16:37:44,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064271 2024-11-11T16:37:44,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064271 2024-11-11T16:37:44,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064271) roll requested 2024-11-11T16:37:44,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064278 2024-11-11T16:37:44,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064278 2024-11-11T16:37:44,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064278) roll requested 2024-11-11T16:37:44,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064285 2024-11-11T16:37:44,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064285 2024-11-11T16:37:44,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064285) roll requested 2024-11-11T16:37:44,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064292 2024-11-11T16:37:44,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064292 2024-11-11T16:37:44,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064292) roll requested 2024-11-11T16:37:44,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064299 2024-11-11T16:37:44,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064299 2024-11-11T16:37:44,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064299) roll requested 2024-11-11T16:37:44,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064306 2024-11-11T16:37:44,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064306 2024-11-11T16:37:44,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064306) roll requested 2024-11-11T16:37:44,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064312 2024-11-11T16:37:44,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064312 2024-11-11T16:37:44,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064312) roll requested 2024-11-11T16:37:44,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064319 2024-11-11T16:37:44,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064319 2024-11-11T16:37:44,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064319) roll requested 2024-11-11T16:37:44,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064326 2024-11-11T16:37:44,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064326 2024-11-11T16:37:44,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064326) roll requested 2024-11-11T16:37:44,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064332 2024-11-11T16:37:44,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064332 2024-11-11T16:37:44,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064332) roll requested 2024-11-11T16:37:44,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064340 2024-11-11T16:37:44,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064340 2024-11-11T16:37:44,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064340) roll requested 2024-11-11T16:37:44,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064346 2024-11-11T16:37:44,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064346 2024-11-11T16:37:44,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064346) roll requested 2024-11-11T16:37:44,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064353 2024-11-11T16:37:44,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064353 2024-11-11T16:37:44,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064353) roll requested 2024-11-11T16:37:44,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064360 2024-11-11T16:37:44,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064360 2024-11-11T16:37:44,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064360) roll requested 2024-11-11T16:37:44,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064368 2024-11-11T16:37:44,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064368 2024-11-11T16:37:44,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064368) roll requested 2024-11-11T16:37:44,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064375 2024-11-11T16:37:44,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064375 2024-11-11T16:37:44,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064375) roll requested 2024-11-11T16:37:44,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064382 2024-11-11T16:37:44,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064382 2024-11-11T16:37:44,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064382) roll requested 2024-11-11T16:37:44,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064393 2024-11-11T16:37:44,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064393 2024-11-11T16:37:44,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064393) roll requested 2024-11-11T16:37:44,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064400 2024-11-11T16:37:44,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064400 2024-11-11T16:37:44,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064400) roll requested 2024-11-11T16:37:44,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064407 2024-11-11T16:37:44,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064407 2024-11-11T16:37:44,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064407) roll requested 2024-11-11T16:37:44,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064414 2024-11-11T16:37:44,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064414 2024-11-11T16:37:44,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064414) roll requested 2024-11-11T16:37:44,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064421 2024-11-11T16:37:44,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064421 2024-11-11T16:37:44,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064421) roll requested 2024-11-11T16:37:44,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064427 2024-11-11T16:37:44,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064427 2024-11-11T16:37:44,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064427) roll requested 2024-11-11T16:37:44,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064433 2024-11-11T16:37:44,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064433 2024-11-11T16:37:44,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064433) roll requested 2024-11-11T16:37:44,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064440 2024-11-11T16:37:44,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064440 2024-11-11T16:37:44,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064440) roll requested 2024-11-11T16:37:44,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064446 2024-11-11T16:37:44,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064446 2024-11-11T16:37:44,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064446) roll requested 2024-11-11T16:37:44,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064455 2024-11-11T16:37:44,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064455 2024-11-11T16:37:44,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064455) roll requested 2024-11-11T16:37:44,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064462 2024-11-11T16:37:44,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064462 2024-11-11T16:37:44,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064462) roll requested 2024-11-11T16:37:44,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064469 2024-11-11T16:37:44,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064469 2024-11-11T16:37:44,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064469) roll requested 2024-11-11T16:37:44,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064476 2024-11-11T16:37:44,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064476 2024-11-11T16:37:44,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064476) roll requested 2024-11-11T16:37:44,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064482 2024-11-11T16:37:44,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064482 2024-11-11T16:37:44,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064482) roll requested 2024-11-11T16:37:44,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064489 2024-11-11T16:37:44,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064489 2024-11-11T16:37:44,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064489) roll requested 2024-11-11T16:37:44,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064497 2024-11-11T16:37:44,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064497 2024-11-11T16:37:44,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064497) roll requested 2024-11-11T16:37:44,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064503 2024-11-11T16:37:44,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064503 2024-11-11T16:37:44,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064503) roll requested 2024-11-11T16:37:44,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064510 2024-11-11T16:37:44,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064510 2024-11-11T16:37:44,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064510) roll requested 2024-11-11T16:37:44,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064517 2024-11-11T16:37:44,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064517 2024-11-11T16:37:44,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064517) roll requested 2024-11-11T16:37:44,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064523 2024-11-11T16:37:44,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064523 2024-11-11T16:37:44,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064523) roll requested 2024-11-11T16:37:44,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064533 2024-11-11T16:37:44,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064533 2024-11-11T16:37:44,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064533) roll requested 2024-11-11T16:37:44,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064545 2024-11-11T16:37:44,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064545 2024-11-11T16:37:44,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064545) roll requested 2024-11-11T16:37:44,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064552 2024-11-11T16:37:44,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064552 2024-11-11T16:37:44,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064552) roll requested 2024-11-11T16:37:44,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064559 2024-11-11T16:37:44,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064559 2024-11-11T16:37:44,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064559) roll requested 2024-11-11T16:37:44,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064565 2024-11-11T16:37:44,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064565 2024-11-11T16:37:44,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064565) roll requested 2024-11-11T16:37:44,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064572 2024-11-11T16:37:44,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064572 2024-11-11T16:37:44,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064572) roll requested 2024-11-11T16:37:44,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064579 2024-11-11T16:37:44,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064579 2024-11-11T16:37:44,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064579) roll requested 2024-11-11T16:37:44,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064585 2024-11-11T16:37:44,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064585 2024-11-11T16:37:44,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064585) roll requested 2024-11-11T16:37:44,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064595 2024-11-11T16:37:44,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064595 2024-11-11T16:37:44,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064595) roll requested 2024-11-11T16:37:44,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064601 2024-11-11T16:37:44,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064601 2024-11-11T16:37:44,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064601) roll requested 2024-11-11T16:37:44,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064608 2024-11-11T16:37:44,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064608 2024-11-11T16:37:44,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064608) roll requested 2024-11-11T16:37:44,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064614 2024-11-11T16:37:44,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064614 2024-11-11T16:37:44,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064614) roll requested 2024-11-11T16:37:44,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064621 2024-11-11T16:37:44,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064621 2024-11-11T16:37:44,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064621) roll requested 2024-11-11T16:37:44,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064627 2024-11-11T16:37:44,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064627 2024-11-11T16:37:44,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064627) roll requested 2024-11-11T16:37:44,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064634 2024-11-11T16:37:44,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064634 2024-11-11T16:37:44,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064627 is not closed yet, will try archiving it next time 2024-11-11T16:37:44,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064634) roll requested 2024-11-11T16:37:44,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064641 2024-11-11T16:37:44,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064641 2024-11-11T16:37:44,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064641) roll requested 2024-11-11T16:37:44,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064649 2024-11-11T16:37:44,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064649 2024-11-11T16:37:44,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064649) roll requested 2024-11-11T16:37:44,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064657 2024-11-11T16:37:44,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064657 2024-11-11T16:37:44,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064657) roll requested 2024-11-11T16:37:44,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064664 2024-11-11T16:37:44,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064664 2024-11-11T16:37:44,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064664) roll requested 2024-11-11T16:37:44,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064672 2024-11-11T16:37:44,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064672 2024-11-11T16:37:44,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064672) roll requested 2024-11-11T16:37:44,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064679 2024-11-11T16:37:44,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064679 2024-11-11T16:37:44,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064679) roll requested 2024-11-11T16:37:44,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064686 2024-11-11T16:37:44,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064686 2024-11-11T16:37:44,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064686) roll requested 2024-11-11T16:37:44,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064692 2024-11-11T16:37:44,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064692 2024-11-11T16:37:44,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064692) roll requested 2024-11-11T16:37:44,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064699 2024-11-11T16:37:44,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064699 2024-11-11T16:37:44,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064699) roll requested 2024-11-11T16:37:44,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064705 2024-11-11T16:37:44,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064705 2024-11-11T16:37:44,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064705) roll requested 2024-11-11T16:37:44,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064711 2024-11-11T16:37:44,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064711 2024-11-11T16:37:44,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064711) roll requested 2024-11-11T16:37:44,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064718 2024-11-11T16:37:44,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064718 2024-11-11T16:37:44,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064718) roll requested 2024-11-11T16:37:44,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064725 2024-11-11T16:37:44,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064725 2024-11-11T16:37:44,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064725) roll requested 2024-11-11T16:37:44,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064732 2024-11-11T16:37:44,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064732 2024-11-11T16:37:44,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064732) roll requested 2024-11-11T16:37:44,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064739 2024-11-11T16:37:44,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064739 2024-11-11T16:37:44,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064739) roll requested 2024-11-11T16:37:44,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064746 2024-11-11T16:37:44,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064746 2024-11-11T16:37:44,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064746) roll requested 2024-11-11T16:37:44,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064753 2024-11-11T16:37:44,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064753 2024-11-11T16:37:44,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064753) roll requested 2024-11-11T16:37:44,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064760 2024-11-11T16:37:44,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064760 2024-11-11T16:37:44,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064760) roll requested 2024-11-11T16:37:44,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064766 2024-11-11T16:37:44,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064766 2024-11-11T16:37:44,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064766) roll requested 2024-11-11T16:37:44,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064773 2024-11-11T16:37:44,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064773 2024-11-11T16:37:44,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064773) roll requested 2024-11-11T16:37:44,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064779 2024-11-11T16:37:44,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064779 2024-11-11T16:37:44,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064779) roll requested 2024-11-11T16:37:44,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064786 2024-11-11T16:37:44,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064786 2024-11-11T16:37:44,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064786) roll requested 2024-11-11T16:37:44,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064793 2024-11-11T16:37:44,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064793 2024-11-11T16:37:44,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064793) roll requested 2024-11-11T16:37:44,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064799 2024-11-11T16:37:44,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064799 2024-11-11T16:37:44,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064799) roll requested 2024-11-11T16:37:44,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064806 2024-11-11T16:37:44,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064806 2024-11-11T16:37:44,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064806) roll requested 2024-11-11T16:37:44,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064813 2024-11-11T16:37:44,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064813 2024-11-11T16:37:44,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064813) roll requested 2024-11-11T16:37:44,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064819 2024-11-11T16:37:44,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064819 2024-11-11T16:37:44,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064819) roll requested 2024-11-11T16:37:44,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064826 2024-11-11T16:37:44,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064826 2024-11-11T16:37:44,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064826) roll requested 2024-11-11T16:37:44,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064832 2024-11-11T16:37:44,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064832 2024-11-11T16:37:44,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064832) roll requested 2024-11-11T16:37:44,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064838 2024-11-11T16:37:44,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064838 2024-11-11T16:37:44,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064838) roll requested 2024-11-11T16:37:44,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064845 2024-11-11T16:37:44,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064845 2024-11-11T16:37:44,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064845) roll requested 2024-11-11T16:37:44,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064851 2024-11-11T16:37:44,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064851 2024-11-11T16:37:44,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064851) roll requested 2024-11-11T16:37:44,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064858 2024-11-11T16:37:44,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064858 2024-11-11T16:37:44,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064858) roll requested 2024-11-11T16:37:44,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064865 2024-11-11T16:37:44,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064865 2024-11-11T16:37:44,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064865) roll requested 2024-11-11T16:37:44,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064871 2024-11-11T16:37:44,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064871 2024-11-11T16:37:44,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064871) roll requested 2024-11-11T16:37:44,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064877 2024-11-11T16:37:44,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064877 2024-11-11T16:37:44,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064877) roll requested 2024-11-11T16:37:44,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064884 2024-11-11T16:37:44,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064884 2024-11-11T16:37:44,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064884) roll requested 2024-11-11T16:37:44,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064891 2024-11-11T16:37:44,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064891 2024-11-11T16:37:44,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064891) roll requested 2024-11-11T16:37:44,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064897 2024-11-11T16:37:44,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064897 2024-11-11T16:37:44,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064897) roll requested 2024-11-11T16:37:44,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064904 2024-11-11T16:37:44,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064904 2024-11-11T16:37:44,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064904) roll requested 2024-11-11T16:37:44,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064911 2024-11-11T16:37:44,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064911 2024-11-11T16:37:44,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064911) roll requested 2024-11-11T16:37:44,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064917 2024-11-11T16:37:44,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064917 2024-11-11T16:37:44,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064917) roll requested 2024-11-11T16:37:44,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064924 2024-11-11T16:37:44,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064924 2024-11-11T16:37:44,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064924) roll requested 2024-11-11T16:37:44,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064931 2024-11-11T16:37:44,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064931 2024-11-11T16:37:44,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064931) roll requested 2024-11-11T16:37:44,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064937 2024-11-11T16:37:44,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064937 2024-11-11T16:37:44,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064937) roll requested 2024-11-11T16:37:44,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064944 2024-11-11T16:37:44,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064944 2024-11-11T16:37:44,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064944) roll requested 2024-11-11T16:37:44,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064950 2024-11-11T16:37:44,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064950 2024-11-11T16:37:44,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064950) roll requested 2024-11-11T16:37:44,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064957 2024-11-11T16:37:44,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064957 2024-11-11T16:37:44,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064957) roll requested 2024-11-11T16:37:44,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064964 2024-11-11T16:37:44,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064964 2024-11-11T16:37:44,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064964) roll requested 2024-11-11T16:37:44,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064970 2024-11-11T16:37:44,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064970 2024-11-11T16:37:44,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064970) roll requested 2024-11-11T16:37:44,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064977 2024-11-11T16:37:44,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064977 2024-11-11T16:37:44,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064977) roll requested 2024-11-11T16:37:44,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064984 2024-11-11T16:37:44,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064984 2024-11-11T16:37:44,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064984) roll requested 2024-11-11T16:37:44,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064991 2024-11-11T16:37:44,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:44,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064991 2024-11-11T16:37:44,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:44,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:44,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064991) roll requested 2024-11-11T16:37:44,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343064997 2024-11-11T16:37:45,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064997 2024-11-11T16:37:45,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343064997) roll requested 2024-11-11T16:37:45,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065004 2024-11-11T16:37:45,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343064997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065004 2024-11-11T16:37:45,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065004) roll requested 2024-11-11T16:37:45,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065010 2024-11-11T16:37:45,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065010 2024-11-11T16:37:45,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065010) roll requested 2024-11-11T16:37:45,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065017 2024-11-11T16:37:45,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065017 2024-11-11T16:37:45,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065017) roll requested 2024-11-11T16:37:45,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065029 2024-11-11T16:37:45,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065029 2024-11-11T16:37:45,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065029) roll requested 2024-11-11T16:37:45,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065036 2024-11-11T16:37:45,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065036 2024-11-11T16:37:45,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065036) roll requested 2024-11-11T16:37:45,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065043 2024-11-11T16:37:45,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065043 2024-11-11T16:37:45,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065043) roll requested 2024-11-11T16:37:45,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065049 2024-11-11T16:37:45,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065049 2024-11-11T16:37:45,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065049) roll requested 2024-11-11T16:37:45,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065056 2024-11-11T16:37:45,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065056 2024-11-11T16:37:45,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065056) roll requested 2024-11-11T16:37:45,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065063 2024-11-11T16:37:45,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065063 2024-11-11T16:37:45,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065063) roll requested 2024-11-11T16:37:45,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065070 2024-11-11T16:37:45,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065070 2024-11-11T16:37:45,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065070) roll requested 2024-11-11T16:37:45,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065077 2024-11-11T16:37:45,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065077 2024-11-11T16:37:45,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065077) roll requested 2024-11-11T16:37:45,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065083 2024-11-11T16:37:45,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065083 2024-11-11T16:37:45,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065083) roll requested 2024-11-11T16:37:45,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065090 2024-11-11T16:37:45,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065090 2024-11-11T16:37:45,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065090) roll requested 2024-11-11T16:37:45,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065096 2024-11-11T16:37:45,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065096 2024-11-11T16:37:45,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065096) roll requested 2024-11-11T16:37:45,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065103 2024-11-11T16:37:45,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065103 2024-11-11T16:37:45,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065103) roll requested 2024-11-11T16:37:45,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065109 2024-11-11T16:37:45,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065109 2024-11-11T16:37:45,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065109) roll requested 2024-11-11T16:37:45,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065116 2024-11-11T16:37:45,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065116 2024-11-11T16:37:45,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065116) roll requested 2024-11-11T16:37:45,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065122 2024-11-11T16:37:45,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065122 2024-11-11T16:37:45,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065122) roll requested 2024-11-11T16:37:45,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065129 2024-11-11T16:37:45,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065129 2024-11-11T16:37:45,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065129) roll requested 2024-11-11T16:37:45,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065135 2024-11-11T16:37:45,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065135 2024-11-11T16:37:45,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065135) roll requested 2024-11-11T16:37:45,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065142 2024-11-11T16:37:45,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065142 2024-11-11T16:37:45,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065142) roll requested 2024-11-11T16:37:45,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065149 2024-11-11T16:37:45,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065149 2024-11-11T16:37:45,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065149) roll requested 2024-11-11T16:37:45,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065165 2024-11-11T16:37:45,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065165 2024-11-11T16:37:45,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065165) roll requested 2024-11-11T16:37:45,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065172 2024-11-11T16:37:45,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065172 2024-11-11T16:37:45,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065172) roll requested 2024-11-11T16:37:45,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065179 2024-11-11T16:37:45,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065179 2024-11-11T16:37:45,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065179) roll requested 2024-11-11T16:37:45,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065186 2024-11-11T16:37:45,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065186 2024-11-11T16:37:45,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065186) roll requested 2024-11-11T16:37:45,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065193 2024-11-11T16:37:45,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065193 2024-11-11T16:37:45,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065193) roll requested 2024-11-11T16:37:45,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065199 2024-11-11T16:37:45,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065199 2024-11-11T16:37:45,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065199) roll requested 2024-11-11T16:37:45,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065206 2024-11-11T16:37:45,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065206 2024-11-11T16:37:45,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065206) roll requested 2024-11-11T16:37:45,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065212 2024-11-11T16:37:45,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065212 2024-11-11T16:37:45,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065212) roll requested 2024-11-11T16:37:45,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065219 2024-11-11T16:37:45,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065219 2024-11-11T16:37:45,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065219) roll requested 2024-11-11T16:37:45,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065225 2024-11-11T16:37:45,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065225 2024-11-11T16:37:45,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065225) roll requested 2024-11-11T16:37:45,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065232 2024-11-11T16:37:45,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065232 2024-11-11T16:37:45,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065232) roll requested 2024-11-11T16:37:45,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065238 2024-11-11T16:37:45,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065238 2024-11-11T16:37:45,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065238) roll requested 2024-11-11T16:37:45,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065245 2024-11-11T16:37:45,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065245 2024-11-11T16:37:45,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065245) roll requested 2024-11-11T16:37:45,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065253 2024-11-11T16:37:45,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065253 2024-11-11T16:37:45,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065253) roll requested 2024-11-11T16:37:45,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065259 2024-11-11T16:37:45,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065259 2024-11-11T16:37:45,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065259) roll requested 2024-11-11T16:37:45,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065266 2024-11-11T16:37:45,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065266 2024-11-11T16:37:45,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065266) roll requested 2024-11-11T16:37:45,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065273 2024-11-11T16:37:45,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065273 2024-11-11T16:37:45,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065273) roll requested 2024-11-11T16:37:45,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065281 2024-11-11T16:37:45,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065281 2024-11-11T16:37:45,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065281) roll requested 2024-11-11T16:37:45,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065287 2024-11-11T16:37:45,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065287 2024-11-11T16:37:45,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065287) roll requested 2024-11-11T16:37:45,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065295 2024-11-11T16:37:45,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065295 2024-11-11T16:37:45,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065295) roll requested 2024-11-11T16:37:45,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065301 2024-11-11T16:37:45,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065301 2024-11-11T16:37:45,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065301) roll requested 2024-11-11T16:37:45,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065308 2024-11-11T16:37:45,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065308 2024-11-11T16:37:45,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065308) roll requested 2024-11-11T16:37:45,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065315 2024-11-11T16:37:45,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065315 2024-11-11T16:37:45,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065315) roll requested 2024-11-11T16:37:45,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065321 2024-11-11T16:37:45,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065321 2024-11-11T16:37:45,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065321) roll requested 2024-11-11T16:37:45,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065328 2024-11-11T16:37:45,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065328 2024-11-11T16:37:45,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065328) roll requested 2024-11-11T16:37:45,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065334 2024-11-11T16:37:45,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065334 2024-11-11T16:37:45,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065334) roll requested 2024-11-11T16:37:45,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065341 2024-11-11T16:37:45,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065341 2024-11-11T16:37:45,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065341) roll requested 2024-11-11T16:37:45,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065347 2024-11-11T16:37:45,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065347 2024-11-11T16:37:45,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065347) roll requested 2024-11-11T16:37:45,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065354 2024-11-11T16:37:45,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065354 2024-11-11T16:37:45,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065354) roll requested 2024-11-11T16:37:45,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065360 2024-11-11T16:37:45,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065360 2024-11-11T16:37:45,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065360) roll requested 2024-11-11T16:37:45,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065367 2024-11-11T16:37:45,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065367 2024-11-11T16:37:45,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065367) roll requested 2024-11-11T16:37:45,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065373 2024-11-11T16:37:45,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065373 2024-11-11T16:37:45,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065373) roll requested 2024-11-11T16:37:45,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065380 2024-11-11T16:37:45,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065380 2024-11-11T16:37:45,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065380) roll requested 2024-11-11T16:37:45,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065387 2024-11-11T16:37:45,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065387 2024-11-11T16:37:45,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065387) roll requested 2024-11-11T16:37:45,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065394 2024-11-11T16:37:45,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065394 2024-11-11T16:37:45,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065394) roll requested 2024-11-11T16:37:45,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065401 2024-11-11T16:37:45,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065401 2024-11-11T16:37:45,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065401) roll requested 2024-11-11T16:37:45,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065408 2024-11-11T16:37:45,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065408 2024-11-11T16:37:45,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065408) roll requested 2024-11-11T16:37:45,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065416 2024-11-11T16:37:45,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065416 2024-11-11T16:37:45,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065416) roll requested 2024-11-11T16:37:45,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065423 2024-11-11T16:37:45,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065423 2024-11-11T16:37:45,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065423) roll requested 2024-11-11T16:37:45,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065430 2024-11-11T16:37:45,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065430 2024-11-11T16:37:45,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065430) roll requested 2024-11-11T16:37:45,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065437 2024-11-11T16:37:45,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065437 2024-11-11T16:37:45,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065437) roll requested 2024-11-11T16:37:45,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065444 2024-11-11T16:37:45,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065444 2024-11-11T16:37:45,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065444) roll requested 2024-11-11T16:37:45,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065451 2024-11-11T16:37:45,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065451 2024-11-11T16:37:45,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065451) roll requested 2024-11-11T16:37:45,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065458 2024-11-11T16:37:45,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065458 2024-11-11T16:37:45,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065458) roll requested 2024-11-11T16:37:45,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065469 2024-11-11T16:37:45,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065469 2024-11-11T16:37:45,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065469) roll requested 2024-11-11T16:37:45,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065477 2024-11-11T16:37:45,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065477 2024-11-11T16:37:45,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065477) roll requested 2024-11-11T16:37:45,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065484 2024-11-11T16:37:45,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065484 2024-11-11T16:37:45,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065484) roll requested 2024-11-11T16:37:45,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065491 2024-11-11T16:37:45,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065491 2024-11-11T16:37:45,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065491) roll requested 2024-11-11T16:37:45,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065499 2024-11-11T16:37:45,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065499 2024-11-11T16:37:45,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065499) roll requested 2024-11-11T16:37:45,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065506 2024-11-11T16:37:45,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065506 2024-11-11T16:37:45,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065506) roll requested 2024-11-11T16:37:45,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065514 2024-11-11T16:37:45,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065514 2024-11-11T16:37:45,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065514) roll requested 2024-11-11T16:37:45,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065522 2024-11-11T16:37:45,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065522 2024-11-11T16:37:45,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065522) roll requested 2024-11-11T16:37:45,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065530 2024-11-11T16:37:45,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065530 2024-11-11T16:37:45,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065530) roll requested 2024-11-11T16:37:45,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065537 2024-11-11T16:37:45,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065537 2024-11-11T16:37:45,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065537) roll requested 2024-11-11T16:37:45,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065548 2024-11-11T16:37:45,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065548 2024-11-11T16:37:45,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065548) roll requested 2024-11-11T16:37:45,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065565 2024-11-11T16:37:45,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065565 2024-11-11T16:37:45,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065565) roll requested 2024-11-11T16:37:45,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065572 2024-11-11T16:37:45,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065572 2024-11-11T16:37:45,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065572) roll requested 2024-11-11T16:37:45,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065581 2024-11-11T16:37:45,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065581 2024-11-11T16:37:45,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065581) roll requested 2024-11-11T16:37:45,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065587 2024-11-11T16:37:45,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065587 2024-11-11T16:37:45,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065587) roll requested 2024-11-11T16:37:45,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065594 2024-11-11T16:37:45,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065594 2024-11-11T16:37:45,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065594) roll requested 2024-11-11T16:37:45,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065601 2024-11-11T16:37:45,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065601 2024-11-11T16:37:45,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065601) roll requested 2024-11-11T16:37:45,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065607 2024-11-11T16:37:45,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065607 2024-11-11T16:37:45,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065607) roll requested 2024-11-11T16:37:45,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065614 2024-11-11T16:37:45,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065614 2024-11-11T16:37:45,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065614) roll requested 2024-11-11T16:37:45,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065621 2024-11-11T16:37:45,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065621 2024-11-11T16:37:45,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065621) roll requested 2024-11-11T16:37:45,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065627 2024-11-11T16:37:45,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065627 2024-11-11T16:37:45,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065627) roll requested 2024-11-11T16:37:45,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065633 2024-11-11T16:37:45,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065633 2024-11-11T16:37:45,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065633) roll requested 2024-11-11T16:37:45,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065640 2024-11-11T16:37:45,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065640 2024-11-11T16:37:45,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065640) roll requested 2024-11-11T16:37:45,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065647 2024-11-11T16:37:45,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065647 2024-11-11T16:37:45,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065647) roll requested 2024-11-11T16:37:45,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065653 2024-11-11T16:37:45,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065653 2024-11-11T16:37:45,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065653) roll requested 2024-11-11T16:37:45,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065660 2024-11-11T16:37:45,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065660 2024-11-11T16:37:45,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065660) roll requested 2024-11-11T16:37:45,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065666 2024-11-11T16:37:45,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065666 2024-11-11T16:37:45,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065666) roll requested 2024-11-11T16:37:45,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065680 2024-11-11T16:37:45,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065680 2024-11-11T16:37:45,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065680) roll requested 2024-11-11T16:37:45,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065691 2024-11-11T16:37:45,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065691 2024-11-11T16:37:45,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065691) roll requested 2024-11-11T16:37:45,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065698 2024-11-11T16:37:45,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065698 2024-11-11T16:37:45,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065698) roll requested 2024-11-11T16:37:45,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065705 2024-11-11T16:37:45,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065705 2024-11-11T16:37:45,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065705) roll requested 2024-11-11T16:37:45,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065711 2024-11-11T16:37:45,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065711 2024-11-11T16:37:45,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065711) roll requested 2024-11-11T16:37:45,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065717 2024-11-11T16:37:45,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065717 2024-11-11T16:37:45,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065717) roll requested 2024-11-11T16:37:45,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065724 2024-11-11T16:37:45,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065724 2024-11-11T16:37:45,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065724) roll requested 2024-11-11T16:37:45,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065734 2024-11-11T16:37:45,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065734 2024-11-11T16:37:45,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065734) roll requested 2024-11-11T16:37:45,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065741 2024-11-11T16:37:45,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065741 2024-11-11T16:37:45,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065741) roll requested 2024-11-11T16:37:45,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065749 2024-11-11T16:37:45,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065749 2024-11-11T16:37:45,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065749) roll requested 2024-11-11T16:37:45,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065755 2024-11-11T16:37:45,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065755 2024-11-11T16:37:45,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065755) roll requested 2024-11-11T16:37:45,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065762 2024-11-11T16:37:45,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065762 2024-11-11T16:37:45,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065762) roll requested 2024-11-11T16:37:45,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065769 2024-11-11T16:37:45,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065769 2024-11-11T16:37:45,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065769) roll requested 2024-11-11T16:37:45,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065778 2024-11-11T16:37:45,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065778 2024-11-11T16:37:45,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065778) roll requested 2024-11-11T16:37:45,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065788 2024-11-11T16:37:45,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065788 2024-11-11T16:37:45,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065788) roll requested 2024-11-11T16:37:45,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065795 2024-11-11T16:37:45,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065795 2024-11-11T16:37:45,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065795) roll requested 2024-11-11T16:37:45,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065802 2024-11-11T16:37:45,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065802 2024-11-11T16:37:45,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065802) roll requested 2024-11-11T16:37:45,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065809 2024-11-11T16:37:45,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065809 2024-11-11T16:37:45,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065809) roll requested 2024-11-11T16:37:45,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065817 2024-11-11T16:37:45,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065817 2024-11-11T16:37:45,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065817) roll requested 2024-11-11T16:37:45,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065823 2024-11-11T16:37:45,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065823 2024-11-11T16:37:45,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065823) roll requested 2024-11-11T16:37:45,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065831 2024-11-11T16:37:45,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065831 2024-11-11T16:37:45,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065831) roll requested 2024-11-11T16:37:45,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065837 2024-11-11T16:37:45,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065837 2024-11-11T16:37:45,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065837) roll requested 2024-11-11T16:37:45,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065846 2024-11-11T16:37:45,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065846 2024-11-11T16:37:45,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065846) roll requested 2024-11-11T16:37:45,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065853 2024-11-11T16:37:45,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065853 2024-11-11T16:37:45,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065853) roll requested 2024-11-11T16:37:45,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065860 2024-11-11T16:37:45,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065860 2024-11-11T16:37:45,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065860) roll requested 2024-11-11T16:37:45,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065869 2024-11-11T16:37:45,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065869 2024-11-11T16:37:45,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065869) roll requested 2024-11-11T16:37:45,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065877 2024-11-11T16:37:45,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065877 2024-11-11T16:37:45,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065877) roll requested 2024-11-11T16:37:45,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065884 2024-11-11T16:37:45,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065884 2024-11-11T16:37:45,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065884) roll requested 2024-11-11T16:37:45,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065893 2024-11-11T16:37:45,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065893 2024-11-11T16:37:45,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065893) roll requested 2024-11-11T16:37:45,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065901 2024-11-11T16:37:45,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065901 2024-11-11T16:37:45,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065901) roll requested 2024-11-11T16:37:45,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065910 2024-11-11T16:37:45,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065910 2024-11-11T16:37:45,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065910) roll requested 2024-11-11T16:37:45,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065917 2024-11-11T16:37:45,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065917 2024-11-11T16:37:45,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065917) roll requested 2024-11-11T16:37:45,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065925 2024-11-11T16:37:45,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065925 2024-11-11T16:37:45,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065925) roll requested 2024-11-11T16:37:45,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065931 2024-11-11T16:37:45,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065931 2024-11-11T16:37:45,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065931) roll requested 2024-11-11T16:37:45,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065938 2024-11-11T16:37:45,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065938 2024-11-11T16:37:45,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065938) roll requested 2024-11-11T16:37:45,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065946 2024-11-11T16:37:45,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065946 2024-11-11T16:37:45,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065946) roll requested 2024-11-11T16:37:45,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065953 2024-11-11T16:37:45,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065953 2024-11-11T16:37:45,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065953) roll requested 2024-11-11T16:37:45,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065961 2024-11-11T16:37:45,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065961 2024-11-11T16:37:45,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065961) roll requested 2024-11-11T16:37:45,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065968 2024-11-11T16:37:45,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065968 2024-11-11T16:37:45,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065968) roll requested 2024-11-11T16:37:45,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065977 2024-11-11T16:37:45,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065977 2024-11-11T16:37:45,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065977) roll requested 2024-11-11T16:37:45,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065984 2024-11-11T16:37:45,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:45,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065984 2024-11-11T16:37:45,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:45,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:45,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065984) roll requested 2024-11-11T16:37:45,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343065994 2024-11-11T16:37:46,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065994 2024-11-11T16:37:46,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343065994) roll requested 2024-11-11T16:37:46,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066001 2024-11-11T16:37:46,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343065994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066001 2024-11-11T16:37:46,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066001) roll requested 2024-11-11T16:37:46,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066008 2024-11-11T16:37:46,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066008 2024-11-11T16:37:46,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066008) roll requested 2024-11-11T16:37:46,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066014 2024-11-11T16:37:46,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066014 2024-11-11T16:37:46,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066014) roll requested 2024-11-11T16:37:46,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066021 2024-11-11T16:37:46,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066021 2024-11-11T16:37:46,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066021) roll requested 2024-11-11T16:37:46,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066028 2024-11-11T16:37:46,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066028 2024-11-11T16:37:46,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066028) roll requested 2024-11-11T16:37:46,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066035 2024-11-11T16:37:46,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066035 2024-11-11T16:37:46,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066035) roll requested 2024-11-11T16:37:46,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066045 2024-11-11T16:37:46,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066045 2024-11-11T16:37:46,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066045) roll requested 2024-11-11T16:37:46,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066052 2024-11-11T16:37:46,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066052 2024-11-11T16:37:46,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066052) roll requested 2024-11-11T16:37:46,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066060 2024-11-11T16:37:46,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066060 2024-11-11T16:37:46,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066060) roll requested 2024-11-11T16:37:46,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066069 2024-11-11T16:37:46,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066069 2024-11-11T16:37:46,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066069) roll requested 2024-11-11T16:37:46,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066075 2024-11-11T16:37:46,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066075 2024-11-11T16:37:46,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066075) roll requested 2024-11-11T16:37:46,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066082 2024-11-11T16:37:46,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066082 2024-11-11T16:37:46,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066082) roll requested 2024-11-11T16:37:46,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066088 2024-11-11T16:37:46,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066088 2024-11-11T16:37:46,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066088) roll requested 2024-11-11T16:37:46,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066095 2024-11-11T16:37:46,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066095 2024-11-11T16:37:46,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066095) roll requested 2024-11-11T16:37:46,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066102 2024-11-11T16:37:46,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066102 2024-11-11T16:37:46,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066102) roll requested 2024-11-11T16:37:46,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066108 2024-11-11T16:37:46,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066108 2024-11-11T16:37:46,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066108) roll requested 2024-11-11T16:37:46,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066114 2024-11-11T16:37:46,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066114 2024-11-11T16:37:46,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066114) roll requested 2024-11-11T16:37:46,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066120 2024-11-11T16:37:46,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066120 2024-11-11T16:37:46,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066120) roll requested 2024-11-11T16:37:46,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066127 2024-11-11T16:37:46,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066127 2024-11-11T16:37:46,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066127) roll requested 2024-11-11T16:37:46,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066133 2024-11-11T16:37:46,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066133 2024-11-11T16:37:46,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066133) roll requested 2024-11-11T16:37:46,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066139 2024-11-11T16:37:46,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066139 2024-11-11T16:37:46,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066139) roll requested 2024-11-11T16:37:46,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066145 2024-11-11T16:37:46,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066145 2024-11-11T16:37:46,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066145) roll requested 2024-11-11T16:37:46,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066152 2024-11-11T16:37:46,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066152 2024-11-11T16:37:46,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066152) roll requested 2024-11-11T16:37:46,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066158 2024-11-11T16:37:46,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066158 2024-11-11T16:37:46,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066158) roll requested 2024-11-11T16:37:46,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066164 2024-11-11T16:37:46,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066164 2024-11-11T16:37:46,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066164) roll requested 2024-11-11T16:37:46,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066171 2024-11-11T16:37:46,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066171 2024-11-11T16:37:46,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066171) roll requested 2024-11-11T16:37:46,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066178 2024-11-11T16:37:46,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066178 2024-11-11T16:37:46,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066178) roll requested 2024-11-11T16:37:46,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066185 2024-11-11T16:37:46,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066185 2024-11-11T16:37:46,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066185) roll requested 2024-11-11T16:37:46,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066192 2024-11-11T16:37:46,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066192 2024-11-11T16:37:46,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066192) roll requested 2024-11-11T16:37:46,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066198 2024-11-11T16:37:46,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066198 2024-11-11T16:37:46,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066198) roll requested 2024-11-11T16:37:46,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066205 2024-11-11T16:37:46,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066205 2024-11-11T16:37:46,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066205) roll requested 2024-11-11T16:37:46,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066211 2024-11-11T16:37:46,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066211 2024-11-11T16:37:46,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066211) roll requested 2024-11-11T16:37:46,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066218 2024-11-11T16:37:46,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066218 2024-11-11T16:37:46,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066218) roll requested 2024-11-11T16:37:46,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066224 2024-11-11T16:37:46,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066224 2024-11-11T16:37:46,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066224) roll requested 2024-11-11T16:37:46,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066233 2024-11-11T16:37:46,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066233 2024-11-11T16:37:46,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066233) roll requested 2024-11-11T16:37:46,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066240 2024-11-11T16:37:46,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066240 2024-11-11T16:37:46,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066240) roll requested 2024-11-11T16:37:46,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066246 2024-11-11T16:37:46,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066246 2024-11-11T16:37:46,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066246) roll requested 2024-11-11T16:37:46,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066253 2024-11-11T16:37:46,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066253 2024-11-11T16:37:46,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066253) roll requested 2024-11-11T16:37:46,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066259 2024-11-11T16:37:46,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066259 2024-11-11T16:37:46,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066259) roll requested 2024-11-11T16:37:46,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066266 2024-11-11T16:37:46,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066266 2024-11-11T16:37:46,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066266) roll requested 2024-11-11T16:37:46,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066273 2024-11-11T16:37:46,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066273 2024-11-11T16:37:46,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066273) roll requested 2024-11-11T16:37:46,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066279 2024-11-11T16:37:46,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066279 2024-11-11T16:37:46,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066279) roll requested 2024-11-11T16:37:46,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066286 2024-11-11T16:37:46,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066286 2024-11-11T16:37:46,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066286) roll requested 2024-11-11T16:37:46,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066293 2024-11-11T16:37:46,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066293 2024-11-11T16:37:46,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066293) roll requested 2024-11-11T16:37:46,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066299 2024-11-11T16:37:46,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066299 2024-11-11T16:37:46,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066299) roll requested 2024-11-11T16:37:46,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066306 2024-11-11T16:37:46,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066306 2024-11-11T16:37:46,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066306) roll requested 2024-11-11T16:37:46,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066312 2024-11-11T16:37:46,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066312 2024-11-11T16:37:46,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066312) roll requested 2024-11-11T16:37:46,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066318 2024-11-11T16:37:46,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066318 2024-11-11T16:37:46,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066318) roll requested 2024-11-11T16:37:46,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066325 2024-11-11T16:37:46,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066325 2024-11-11T16:37:46,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066325) roll requested 2024-11-11T16:37:46,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066336 2024-11-11T16:37:46,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066336 2024-11-11T16:37:46,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066336) roll requested 2024-11-11T16:37:46,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066344 2024-11-11T16:37:46,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066344 2024-11-11T16:37:46,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066344) roll requested 2024-11-11T16:37:46,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066351 2024-11-11T16:37:46,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066351 2024-11-11T16:37:46,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066351) roll requested 2024-11-11T16:37:46,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066366 2024-11-11T16:37:46,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066366 2024-11-11T16:37:46,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066366) roll requested 2024-11-11T16:37:46,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066374 2024-11-11T16:37:46,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066374 2024-11-11T16:37:46,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066374) roll requested 2024-11-11T16:37:46,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066381 2024-11-11T16:37:46,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066381 2024-11-11T16:37:46,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066381) roll requested 2024-11-11T16:37:46,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066395 2024-11-11T16:37:46,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066395 2024-11-11T16:37:46,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066395) roll requested 2024-11-11T16:37:46,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066404 2024-11-11T16:37:46,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066404 2024-11-11T16:37:46,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066404) roll requested 2024-11-11T16:37:46,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066428 2024-11-11T16:37:46,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066428 2024-11-11T16:37:46,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066428) roll requested 2024-11-11T16:37:46,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066437 2024-11-11T16:37:46,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066437 2024-11-11T16:37:46,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066437) roll requested 2024-11-11T16:37:46,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066444 2024-11-11T16:37:46,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066444 2024-11-11T16:37:46,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066444) roll requested 2024-11-11T16:37:46,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066451 2024-11-11T16:37:46,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066451 2024-11-11T16:37:46,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066451) roll requested 2024-11-11T16:37:46,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066461 2024-11-11T16:37:46,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066461 2024-11-11T16:37:46,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066461) roll requested 2024-11-11T16:37:46,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066468 2024-11-11T16:37:46,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066468 2024-11-11T16:37:46,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066468) roll requested 2024-11-11T16:37:46,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066474 2024-11-11T16:37:46,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066474 2024-11-11T16:37:46,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066474) roll requested 2024-11-11T16:37:46,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066481 2024-11-11T16:37:46,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066481 2024-11-11T16:37:46,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066481) roll requested 2024-11-11T16:37:46,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066487 2024-11-11T16:37:46,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066487 2024-11-11T16:37:46,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066487) roll requested 2024-11-11T16:37:46,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066493 2024-11-11T16:37:46,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066493 2024-11-11T16:37:46,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066493) roll requested 2024-11-11T16:37:46,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066500 2024-11-11T16:37:46,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066500 2024-11-11T16:37:46,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066500) roll requested 2024-11-11T16:37:46,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066507 2024-11-11T16:37:46,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066507 2024-11-11T16:37:46,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066507) roll requested 2024-11-11T16:37:46,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066513 2024-11-11T16:37:46,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066513 2024-11-11T16:37:46,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066513) roll requested 2024-11-11T16:37:46,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066520 2024-11-11T16:37:46,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066520 2024-11-11T16:37:46,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066520) roll requested 2024-11-11T16:37:46,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066526 2024-11-11T16:37:46,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066526 2024-11-11T16:37:46,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066526) roll requested 2024-11-11T16:37:46,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066532 2024-11-11T16:37:46,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066532 2024-11-11T16:37:46,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066532) roll requested 2024-11-11T16:37:46,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066539 2024-11-11T16:37:46,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066539 2024-11-11T16:37:46,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066539) roll requested 2024-11-11T16:37:46,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066545 2024-11-11T16:37:46,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066545 2024-11-11T16:37:46,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066545) roll requested 2024-11-11T16:37:46,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066552 2024-11-11T16:37:46,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066552 2024-11-11T16:37:46,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066552) roll requested 2024-11-11T16:37:46,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066559 2024-11-11T16:37:46,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066559 2024-11-11T16:37:46,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066559) roll requested 2024-11-11T16:37:46,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066565 2024-11-11T16:37:46,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066565 2024-11-11T16:37:46,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066565) roll requested 2024-11-11T16:37:46,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066576 2024-11-11T16:37:46,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066576 2024-11-11T16:37:46,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066565 is not closed yet, will try archiving it next time 2024-11-11T16:37:46,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066576) roll requested 2024-11-11T16:37:46,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066585 2024-11-11T16:37:46,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066585 2024-11-11T16:37:46,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066585) roll requested 2024-11-11T16:37:46,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066593 2024-11-11T16:37:46,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066593 2024-11-11T16:37:46,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066593) roll requested 2024-11-11T16:37:46,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066600 2024-11-11T16:37:46,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066600 2024-11-11T16:37:46,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066600) roll requested 2024-11-11T16:37:46,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066608 2024-11-11T16:37:46,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066600 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066608 2024-11-11T16:37:46,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066608) roll requested 2024-11-11T16:37:46,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066616 2024-11-11T16:37:46,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066616 2024-11-11T16:37:46,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066616) roll requested 2024-11-11T16:37:46,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066623 2024-11-11T16:37:46,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066623 2024-11-11T16:37:46,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066623) roll requested 2024-11-11T16:37:46,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066629 2024-11-11T16:37:46,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066629 2024-11-11T16:37:46,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066629) roll requested 2024-11-11T16:37:46,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066636 2024-11-11T16:37:46,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066636 2024-11-11T16:37:46,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066636) roll requested 2024-11-11T16:37:46,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066642 2024-11-11T16:37:46,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066642 2024-11-11T16:37:46,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066642) roll requested 2024-11-11T16:37:46,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066651 2024-11-11T16:37:46,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066651 2024-11-11T16:37:46,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066651) roll requested 2024-11-11T16:37:46,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066660 2024-11-11T16:37:46,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066660 2024-11-11T16:37:46,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066660) roll requested 2024-11-11T16:37:46,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066668 2024-11-11T16:37:46,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066668 2024-11-11T16:37:46,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066668) roll requested 2024-11-11T16:37:46,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066675 2024-11-11T16:37:46,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066675 2024-11-11T16:37:46,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066675) roll requested 2024-11-11T16:37:46,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066685 2024-11-11T16:37:46,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066685 2024-11-11T16:37:46,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066685) roll requested 2024-11-11T16:37:46,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066693 2024-11-11T16:37:46,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066693 2024-11-11T16:37:46,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066693) roll requested 2024-11-11T16:37:46,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066701 2024-11-11T16:37:46,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066701 2024-11-11T16:37:46,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066701) roll requested 2024-11-11T16:37:46,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066709 2024-11-11T16:37:46,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066709 2024-11-11T16:37:46,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066709) roll requested 2024-11-11T16:37:46,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066721 2024-11-11T16:37:46,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066721 2024-11-11T16:37:46,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066721) roll requested 2024-11-11T16:37:46,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066737 2024-11-11T16:37:46,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066737 2024-11-11T16:37:46,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066737) roll requested 2024-11-11T16:37:46,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066761 2024-11-11T16:37:46,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066761 2024-11-11T16:37:46,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066761) roll requested 2024-11-11T16:37:46,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066793 2024-11-11T16:37:46,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066793 2024-11-11T16:37:46,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066793) roll requested 2024-11-11T16:37:46,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066809 2024-11-11T16:37:46,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066809 2024-11-11T16:37:46,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066809) roll requested 2024-11-11T16:37:46,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066845 2024-11-11T16:37:46,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066845 2024-11-11T16:37:46,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066845) roll requested 2024-11-11T16:37:46,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066860 2024-11-11T16:37:46,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066860 2024-11-11T16:37:46,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066860) roll requested 2024-11-11T16:37:46,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066893 2024-11-11T16:37:46,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066893 2024-11-11T16:37:46,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066893) roll requested 2024-11-11T16:37:46,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066947 2024-11-11T16:37:46,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:46,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066947 2024-11-11T16:37:46,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:46,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:46,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066947) roll requested 2024-11-11T16:37:46,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343066993 2024-11-11T16:37:47,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066993 2024-11-11T16:37:47,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343066993) roll requested 2024-11-11T16:37:47,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067055 2024-11-11T16:37:47,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343066993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067055 2024-11-11T16:37:47,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067055) roll requested 2024-11-11T16:37:47,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067085 2024-11-11T16:37:47,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067085 2024-11-11T16:37:47,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067085) roll requested 2024-11-11T16:37:47,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067094 2024-11-11T16:37:47,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067094 2024-11-11T16:37:47,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067094) roll requested 2024-11-11T16:37:47,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067106 2024-11-11T16:37:47,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067106 2024-11-11T16:37:47,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067106) roll requested 2024-11-11T16:37:47,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067114 2024-11-11T16:37:47,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067114 2024-11-11T16:37:47,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067114) roll requested 2024-11-11T16:37:47,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067130 2024-11-11T16:37:47,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067130 2024-11-11T16:37:47,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067130) roll requested 2024-11-11T16:37:47,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067145 2024-11-11T16:37:47,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067145 2024-11-11T16:37:47,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067145) roll requested 2024-11-11T16:37:47,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067156 2024-11-11T16:37:47,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067156 2024-11-11T16:37:47,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067156) roll requested 2024-11-11T16:37:47,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067165 2024-11-11T16:37:47,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067165 2024-11-11T16:37:47,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067165) roll requested 2024-11-11T16:37:47,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067172 2024-11-11T16:37:47,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067172 2024-11-11T16:37:47,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067172) roll requested 2024-11-11T16:37:47,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067188 2024-11-11T16:37:47,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067188 2024-11-11T16:37:47,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067188) roll requested 2024-11-11T16:37:47,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067198 2024-11-11T16:37:47,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067198 2024-11-11T16:37:47,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067198) roll requested 2024-11-11T16:37:47,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067206 2024-11-11T16:37:47,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067206 2024-11-11T16:37:47,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067206) roll requested 2024-11-11T16:37:47,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067213 2024-11-11T16:37:47,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067213 2024-11-11T16:37:47,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067213) roll requested 2024-11-11T16:37:47,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067220 2024-11-11T16:37:47,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067220 2024-11-11T16:37:47,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067220) roll requested 2024-11-11T16:37:47,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067227 2024-11-11T16:37:47,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067227 2024-11-11T16:37:47,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067227) roll requested 2024-11-11T16:37:47,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067233 2024-11-11T16:37:47,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067233 2024-11-11T16:37:47,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067233) roll requested 2024-11-11T16:37:47,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067239 2024-11-11T16:37:47,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067239 2024-11-11T16:37:47,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067239) roll requested 2024-11-11T16:37:47,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067247 2024-11-11T16:37:47,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067247 2024-11-11T16:37:47,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067247) roll requested 2024-11-11T16:37:47,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067253 2024-11-11T16:37:47,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067253 2024-11-11T16:37:47,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067253) roll requested 2024-11-11T16:37:47,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067261 2024-11-11T16:37:47,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067261 2024-11-11T16:37:47,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067261) roll requested 2024-11-11T16:37:47,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067268 2024-11-11T16:37:47,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067268 2024-11-11T16:37:47,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067268) roll requested 2024-11-11T16:37:47,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067275 2024-11-11T16:37:47,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067275 2024-11-11T16:37:47,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067275) roll requested 2024-11-11T16:37:47,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067282 2024-11-11T16:37:47,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067282 2024-11-11T16:37:47,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067282) roll requested 2024-11-11T16:37:47,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067290 2024-11-11T16:37:47,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067290 2024-11-11T16:37:47,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067290) roll requested 2024-11-11T16:37:47,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067296 2024-11-11T16:37:47,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067296 2024-11-11T16:37:47,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067296) roll requested 2024-11-11T16:37:47,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067303 2024-11-11T16:37:47,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067303 2024-11-11T16:37:47,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067303) roll requested 2024-11-11T16:37:47,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067311 2024-11-11T16:37:47,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067311 2024-11-11T16:37:47,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067311) roll requested 2024-11-11T16:37:47,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067318 2024-11-11T16:37:47,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067318 2024-11-11T16:37:47,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067318) roll requested 2024-11-11T16:37:47,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067325 2024-11-11T16:37:47,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067325 2024-11-11T16:37:47,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067325) roll requested 2024-11-11T16:37:47,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067332 2024-11-11T16:37:47,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067332 2024-11-11T16:37:47,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067332) roll requested 2024-11-11T16:37:47,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067340 2024-11-11T16:37:47,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067340 2024-11-11T16:37:47,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067340) roll requested 2024-11-11T16:37:47,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067347 2024-11-11T16:37:47,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067347 2024-11-11T16:37:47,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067347) roll requested 2024-11-11T16:37:47,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067354 2024-11-11T16:37:47,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067354 2024-11-11T16:37:47,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067354) roll requested 2024-11-11T16:37:47,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067362 2024-11-11T16:37:47,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067362 2024-11-11T16:37:47,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067362) roll requested 2024-11-11T16:37:47,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067369 2024-11-11T16:37:47,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067369 2024-11-11T16:37:47,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067369) roll requested 2024-11-11T16:37:47,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067376 2024-11-11T16:37:47,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067376 2024-11-11T16:37:47,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067376) roll requested 2024-11-11T16:37:47,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067382 2024-11-11T16:37:47,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067382 2024-11-11T16:37:47,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067382) roll requested 2024-11-11T16:37:47,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067389 2024-11-11T16:37:47,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067389 2024-11-11T16:37:47,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067389) roll requested 2024-11-11T16:37:47,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067396 2024-11-11T16:37:47,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067396 2024-11-11T16:37:47,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067396) roll requested 2024-11-11T16:37:47,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067403 2024-11-11T16:37:47,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067403 2024-11-11T16:37:47,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067403) roll requested 2024-11-11T16:37:47,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067410 2024-11-11T16:37:47,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067410 2024-11-11T16:37:47,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067410) roll requested 2024-11-11T16:37:47,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067417 2024-11-11T16:37:47,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067417 2024-11-11T16:37:47,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067417) roll requested 2024-11-11T16:37:47,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067424 2024-11-11T16:37:47,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067424 2024-11-11T16:37:47,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067424) roll requested 2024-11-11T16:37:47,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067431 2024-11-11T16:37:47,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067431 2024-11-11T16:37:47,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067431) roll requested 2024-11-11T16:37:47,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067438 2024-11-11T16:37:47,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067438 2024-11-11T16:37:47,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067431 is not closed yet, will try archiving it next time 2024-11-11T16:37:47,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067438) roll requested 2024-11-11T16:37:47,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067445 2024-11-11T16:37:47,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067445 2024-11-11T16:37:47,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067445) roll requested 2024-11-11T16:37:47,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067452 2024-11-11T16:37:47,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067452 2024-11-11T16:37:47,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067452) roll requested 2024-11-11T16:37:47,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067459 2024-11-11T16:37:47,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067459 2024-11-11T16:37:47,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067459) roll requested 2024-11-11T16:37:47,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067466 2024-11-11T16:37:47,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067466 2024-11-11T16:37:47,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067466) roll requested 2024-11-11T16:37:47,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067473 2024-11-11T16:37:47,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067473 2024-11-11T16:37:47,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067473) roll requested 2024-11-11T16:37:47,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067480 2024-11-11T16:37:47,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067480 2024-11-11T16:37:47,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067480) roll requested 2024-11-11T16:37:47,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067487 2024-11-11T16:37:47,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067487 2024-11-11T16:37:47,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067487) roll requested 2024-11-11T16:37:47,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067494 2024-11-11T16:37:47,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067494 2024-11-11T16:37:47,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067494) roll requested 2024-11-11T16:37:47,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067501 2024-11-11T16:37:47,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067501 2024-11-11T16:37:47,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067501) roll requested 2024-11-11T16:37:47,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067508 2024-11-11T16:37:47,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067508 2024-11-11T16:37:47,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067508) roll requested 2024-11-11T16:37:47,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067515 2024-11-11T16:37:47,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067515 2024-11-11T16:37:47,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067515) roll requested 2024-11-11T16:37:47,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067522 2024-11-11T16:37:47,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067522 2024-11-11T16:37:47,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,529 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067522) roll requested 2024-11-11T16:37:47,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067529 2024-11-11T16:37:47,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067529 2024-11-11T16:37:47,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067529) roll requested 2024-11-11T16:37:47,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067537 2024-11-11T16:37:47,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067537 2024-11-11T16:37:47,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067537) roll requested 2024-11-11T16:37:47,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067544 2024-11-11T16:37:47,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067544 2024-11-11T16:37:47,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067544) roll requested 2024-11-11T16:37:47,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067551 2024-11-11T16:37:47,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067551 2024-11-11T16:37:47,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067551) roll requested 2024-11-11T16:37:47,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067559 2024-11-11T16:37:47,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067559 2024-11-11T16:37:47,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067559) roll requested 2024-11-11T16:37:47,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067566 2024-11-11T16:37:47,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067566 2024-11-11T16:37:47,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067566) roll requested 2024-11-11T16:37:47,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067573 2024-11-11T16:37:47,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067573 2024-11-11T16:37:47,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067573) roll requested 2024-11-11T16:37:47,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067580 2024-11-11T16:37:47,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067580 2024-11-11T16:37:47,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067580) roll requested 2024-11-11T16:37:47,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067589 2024-11-11T16:37:47,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067589 2024-11-11T16:37:47,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067589) roll requested 2024-11-11T16:37:47,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067597 2024-11-11T16:37:47,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067597 2024-11-11T16:37:47,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067597) roll requested 2024-11-11T16:37:47,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067604 2024-11-11T16:37:47,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067604 2024-11-11T16:37:47,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067604) roll requested 2024-11-11T16:37:47,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067613 2024-11-11T16:37:47,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067613 2024-11-11T16:37:47,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067613) roll requested 2024-11-11T16:37:47,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067632 2024-11-11T16:37:47,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067632 2024-11-11T16:37:47,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067632) roll requested 2024-11-11T16:37:47,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067638 2024-11-11T16:37:47,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067638 2024-11-11T16:37:47,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067638) roll requested 2024-11-11T16:37:47,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067645 2024-11-11T16:37:47,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067645 2024-11-11T16:37:47,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067645) roll requested 2024-11-11T16:37:47,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067652 2024-11-11T16:37:47,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067652 2024-11-11T16:37:47,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067652) roll requested 2024-11-11T16:37:47,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067660 2024-11-11T16:37:47,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067660 2024-11-11T16:37:47,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067660) roll requested 2024-11-11T16:37:47,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067667 2024-11-11T16:37:47,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067667 2024-11-11T16:37:47,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067667) roll requested 2024-11-11T16:37:47,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067676 2024-11-11T16:37:47,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067676 2024-11-11T16:37:47,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067676) roll requested 2024-11-11T16:37:47,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067689 2024-11-11T16:37:47,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067689 2024-11-11T16:37:47,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067689) roll requested 2024-11-11T16:37:47,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067697 2024-11-11T16:37:47,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067697 2024-11-11T16:37:47,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067697) roll requested 2024-11-11T16:37:47,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067707 2024-11-11T16:37:47,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067707 2024-11-11T16:37:47,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067707) roll requested 2024-11-11T16:37:47,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067713 2024-11-11T16:37:47,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067713 2024-11-11T16:37:47,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067713) roll requested 2024-11-11T16:37:47,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067721 2024-11-11T16:37:47,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067721 2024-11-11T16:37:47,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067721) roll requested 2024-11-11T16:37:47,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067728 2024-11-11T16:37:47,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067728 2024-11-11T16:37:47,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067728) roll requested 2024-11-11T16:37:47,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067734 2024-11-11T16:37:47,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067734 2024-11-11T16:37:47,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067734) roll requested 2024-11-11T16:37:47,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067741 2024-11-11T16:37:47,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067741 2024-11-11T16:37:47,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067741) roll requested 2024-11-11T16:37:47,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067747 2024-11-11T16:37:47,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067747 2024-11-11T16:37:47,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067747) roll requested 2024-11-11T16:37:47,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067754 2024-11-11T16:37:47,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067754 2024-11-11T16:37:47,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067754) roll requested 2024-11-11T16:37:47,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067761 2024-11-11T16:37:47,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067761 2024-11-11T16:37:47,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067761) roll requested 2024-11-11T16:37:47,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067767 2024-11-11T16:37:47,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067767 2024-11-11T16:37:47,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067767) roll requested 2024-11-11T16:37:47,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067773 2024-11-11T16:37:47,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067773 2024-11-11T16:37:47,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067773) roll requested 2024-11-11T16:37:47,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067779 2024-11-11T16:37:47,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067779 2024-11-11T16:37:47,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067779) roll requested 2024-11-11T16:37:47,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067785 2024-11-11T16:37:47,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067785 2024-11-11T16:37:47,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067785) roll requested 2024-11-11T16:37:47,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067792 2024-11-11T16:37:47,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067792 2024-11-11T16:37:47,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067792) roll requested 2024-11-11T16:37:47,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067798 2024-11-11T16:37:47,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067798 2024-11-11T16:37:47,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067798) roll requested 2024-11-11T16:37:47,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067805 2024-11-11T16:37:47,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067805 2024-11-11T16:37:47,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067805) roll requested 2024-11-11T16:37:47,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067811 2024-11-11T16:37:47,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067811 2024-11-11T16:37:47,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067811) roll requested 2024-11-11T16:37:47,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067817 2024-11-11T16:37:47,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067817 2024-11-11T16:37:47,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067817) roll requested 2024-11-11T16:37:47,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067823 2024-11-11T16:37:47,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067823 2024-11-11T16:37:47,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067823) roll requested 2024-11-11T16:37:47,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067830 2024-11-11T16:37:47,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067830 2024-11-11T16:37:47,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067830) roll requested 2024-11-11T16:37:47,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067839 2024-11-11T16:37:47,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067839 2024-11-11T16:37:47,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067839) roll requested 2024-11-11T16:37:47,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067854 2024-11-11T16:37:47,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067854 2024-11-11T16:37:47,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067854) roll requested 2024-11-11T16:37:47,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067861 2024-11-11T16:37:47,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067861 2024-11-11T16:37:47,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067861) roll requested 2024-11-11T16:37:47,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067868 2024-11-11T16:37:47,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067868 2024-11-11T16:37:47,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067868) roll requested 2024-11-11T16:37:47,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067874 2024-11-11T16:37:47,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067874 2024-11-11T16:37:47,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067874) roll requested 2024-11-11T16:37:47,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067880 2024-11-11T16:37:47,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067880 2024-11-11T16:37:47,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067880) roll requested 2024-11-11T16:37:47,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067887 2024-11-11T16:37:47,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067887 2024-11-11T16:37:47,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067887) roll requested 2024-11-11T16:37:47,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067893 2024-11-11T16:37:47,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067893 2024-11-11T16:37:47,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067893) roll requested 2024-11-11T16:37:47,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067900 2024-11-11T16:37:47,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067900 2024-11-11T16:37:47,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067900) roll requested 2024-11-11T16:37:47,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067906 2024-11-11T16:37:47,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067906 2024-11-11T16:37:47,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067906) roll requested 2024-11-11T16:37:47,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067912 2024-11-11T16:37:47,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067912 2024-11-11T16:37:47,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067912) roll requested 2024-11-11T16:37:47,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067918 2024-11-11T16:37:47,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067918 2024-11-11T16:37:47,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067918) roll requested 2024-11-11T16:37:47,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067925 2024-11-11T16:37:47,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067925 2024-11-11T16:37:47,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067925) roll requested 2024-11-11T16:37:47,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067931 2024-11-11T16:37:47,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067931 2024-11-11T16:37:47,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067931) roll requested 2024-11-11T16:37:47,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067937 2024-11-11T16:37:47,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067937 2024-11-11T16:37:47,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067937) roll requested 2024-11-11T16:37:47,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067944 2024-11-11T16:37:47,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067944 2024-11-11T16:37:47,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067944) roll requested 2024-11-11T16:37:47,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067951 2024-11-11T16:37:47,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067951 2024-11-11T16:37:47,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067951) roll requested 2024-11-11T16:37:47,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067958 2024-11-11T16:37:47,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067958 2024-11-11T16:37:47,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067958) roll requested 2024-11-11T16:37:47,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067965 2024-11-11T16:37:47,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067965 2024-11-11T16:37:47,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067965) roll requested 2024-11-11T16:37:47,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067973 2024-11-11T16:37:47,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067973 2024-11-11T16:37:47,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067973) roll requested 2024-11-11T16:37:47,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067980 2024-11-11T16:37:47,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067980 2024-11-11T16:37:47,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067980) roll requested 2024-11-11T16:37:47,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067987 2024-11-11T16:37:47,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:47,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067987 2024-11-11T16:37:47,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:47,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:47,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067987) roll requested 2024-11-11T16:37:47,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343067995 2024-11-11T16:37:48,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067995 2024-11-11T16:37:48,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343067995) roll requested 2024-11-11T16:37:48,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068002 2024-11-11T16:37:48,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343067995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068002 2024-11-11T16:37:48,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068002) roll requested 2024-11-11T16:37:48,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068010 2024-11-11T16:37:48,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068010 2024-11-11T16:37:48,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068010) roll requested 2024-11-11T16:37:48,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068025 2024-11-11T16:37:48,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068025 2024-11-11T16:37:48,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068025) roll requested 2024-11-11T16:37:48,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068036 2024-11-11T16:37:48,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068036 2024-11-11T16:37:48,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068036) roll requested 2024-11-11T16:37:48,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068047 2024-11-11T16:37:48,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068047 2024-11-11T16:37:48,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068047) roll requested 2024-11-11T16:37:48,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068055 2024-11-11T16:37:48,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068055 2024-11-11T16:37:48,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068055) roll requested 2024-11-11T16:37:48,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068062 2024-11-11T16:37:48,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068062 2024-11-11T16:37:48,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068062) roll requested 2024-11-11T16:37:48,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068071 2024-11-11T16:37:48,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068071 2024-11-11T16:37:48,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068071) roll requested 2024-11-11T16:37:48,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068079 2024-11-11T16:37:48,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068079 2024-11-11T16:37:48,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068079) roll requested 2024-11-11T16:37:48,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068091 2024-11-11T16:37:48,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068091 2024-11-11T16:37:48,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068091) roll requested 2024-11-11T16:37:48,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068105 2024-11-11T16:37:48,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068105 2024-11-11T16:37:48,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068105) roll requested 2024-11-11T16:37:48,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068120 2024-11-11T16:37:48,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068120 2024-11-11T16:37:48,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068120) roll requested 2024-11-11T16:37:48,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068129 2024-11-11T16:37:48,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068129 2024-11-11T16:37:48,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068129) roll requested 2024-11-11T16:37:48,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068137 2024-11-11T16:37:48,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068137 2024-11-11T16:37:48,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068137) roll requested 2024-11-11T16:37:48,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068145 2024-11-11T16:37:48,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068145 2024-11-11T16:37:48,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068145) roll requested 2024-11-11T16:37:48,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068153 2024-11-11T16:37:48,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068153 2024-11-11T16:37:48,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068153) roll requested 2024-11-11T16:37:48,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068160 2024-11-11T16:37:48,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068160 2024-11-11T16:37:48,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068160) roll requested 2024-11-11T16:37:48,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068169 2024-11-11T16:37:48,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068169 2024-11-11T16:37:48,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068169) roll requested 2024-11-11T16:37:48,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068177 2024-11-11T16:37:48,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068177 2024-11-11T16:37:48,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068177) roll requested 2024-11-11T16:37:48,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068184 2024-11-11T16:37:48,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068184 2024-11-11T16:37:48,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068184) roll requested 2024-11-11T16:37:48,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068192 2024-11-11T16:37:48,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068192 2024-11-11T16:37:48,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068192) roll requested 2024-11-11T16:37:48,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068201 2024-11-11T16:37:48,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068201 2024-11-11T16:37:48,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068201) roll requested 2024-11-11T16:37:48,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068209 2024-11-11T16:37:48,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068209 2024-11-11T16:37:48,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068209) roll requested 2024-11-11T16:37:48,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068217 2024-11-11T16:37:48,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068217 2024-11-11T16:37:48,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068217) roll requested 2024-11-11T16:37:48,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068229 2024-11-11T16:37:48,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068229 2024-11-11T16:37:48,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068229) roll requested 2024-11-11T16:37:48,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068238 2024-11-11T16:37:48,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068238 2024-11-11T16:37:48,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068238) roll requested 2024-11-11T16:37:48,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068249 2024-11-11T16:37:48,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068249 2024-11-11T16:37:48,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068249) roll requested 2024-11-11T16:37:48,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068257 2024-11-11T16:37:48,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068257 2024-11-11T16:37:48,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068257) roll requested 2024-11-11T16:37:48,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068264 2024-11-11T16:37:48,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068264 2024-11-11T16:37:48,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068264) roll requested 2024-11-11T16:37:48,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068270 2024-11-11T16:37:48,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068270 2024-11-11T16:37:48,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068270) roll requested 2024-11-11T16:37:48,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068278 2024-11-11T16:37:48,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068278 2024-11-11T16:37:48,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068278) roll requested 2024-11-11T16:37:48,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068286 2024-11-11T16:37:48,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068286 2024-11-11T16:37:48,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068286) roll requested 2024-11-11T16:37:48,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068298 2024-11-11T16:37:48,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068298 2024-11-11T16:37:48,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068298) roll requested 2024-11-11T16:37:48,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068318 2024-11-11T16:37:48,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068318 2024-11-11T16:37:48,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068318) roll requested 2024-11-11T16:37:48,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068333 2024-11-11T16:37:48,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068333 2024-11-11T16:37:48,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068333) roll requested 2024-11-11T16:37:48,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068353 2024-11-11T16:37:48,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068353 2024-11-11T16:37:48,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068353) roll requested 2024-11-11T16:37:48,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068365 2024-11-11T16:37:48,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068365 2024-11-11T16:37:48,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068365) roll requested 2024-11-11T16:37:48,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068373 2024-11-11T16:37:48,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068373 2024-11-11T16:37:48,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068373) roll requested 2024-11-11T16:37:48,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068380 2024-11-11T16:37:48,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068380 2024-11-11T16:37:48,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068380) roll requested 2024-11-11T16:37:48,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068386 2024-11-11T16:37:48,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068386 2024-11-11T16:37:48,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068386) roll requested 2024-11-11T16:37:48,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068393 2024-11-11T16:37:48,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068393 2024-11-11T16:37:48,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068393) roll requested 2024-11-11T16:37:48,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068401 2024-11-11T16:37:48,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068401 2024-11-11T16:37:48,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068401) roll requested 2024-11-11T16:37:48,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068408 2024-11-11T16:37:48,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068408 2024-11-11T16:37:48,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068408) roll requested 2024-11-11T16:37:48,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068414 2024-11-11T16:37:48,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068414 2024-11-11T16:37:48,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068414) roll requested 2024-11-11T16:37:48,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068422 2024-11-11T16:37:48,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068422 2024-11-11T16:37:48,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068422) roll requested 2024-11-11T16:37:48,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068428 2024-11-11T16:37:48,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068428 2024-11-11T16:37:48,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068428) roll requested 2024-11-11T16:37:48,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068435 2024-11-11T16:37:48,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068435 2024-11-11T16:37:48,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068435) roll requested 2024-11-11T16:37:48,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068442 2024-11-11T16:37:48,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068442 2024-11-11T16:37:48,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068442) roll requested 2024-11-11T16:37:48,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068450 2024-11-11T16:37:48,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068450 2024-11-11T16:37:48,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068450) roll requested 2024-11-11T16:37:48,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068458 2024-11-11T16:37:48,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068458 2024-11-11T16:37:48,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068458) roll requested 2024-11-11T16:37:48,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068467 2024-11-11T16:37:48,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068467 2024-11-11T16:37:48,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068467) roll requested 2024-11-11T16:37:48,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068475 2024-11-11T16:37:48,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068475 2024-11-11T16:37:48,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068475) roll requested 2024-11-11T16:37:48,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068484 2024-11-11T16:37:48,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068484 2024-11-11T16:37:48,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068484) roll requested 2024-11-11T16:37:48,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068493 2024-11-11T16:37:48,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068493 2024-11-11T16:37:48,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068493) roll requested 2024-11-11T16:37:48,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068505 2024-11-11T16:37:48,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068505 2024-11-11T16:37:48,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068505) roll requested 2024-11-11T16:37:48,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068513 2024-11-11T16:37:48,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068513 2024-11-11T16:37:48,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068513) roll requested 2024-11-11T16:37:48,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068522 2024-11-11T16:37:48,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068522 2024-11-11T16:37:48,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,529 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068522) roll requested 2024-11-11T16:37:48,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068530 2024-11-11T16:37:48,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068530 2024-11-11T16:37:48,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068530) roll requested 2024-11-11T16:37:48,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068541 2024-11-11T16:37:48,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068541 2024-11-11T16:37:48,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068541) roll requested 2024-11-11T16:37:48,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068551 2024-11-11T16:37:48,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068551 2024-11-11T16:37:48,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068551) roll requested 2024-11-11T16:37:48,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068562 2024-11-11T16:37:48,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068562 2024-11-11T16:37:48,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068562) roll requested 2024-11-11T16:37:48,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068570 2024-11-11T16:37:48,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068570 2024-11-11T16:37:48,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068570) roll requested 2024-11-11T16:37:48,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068581 2024-11-11T16:37:48,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068581 2024-11-11T16:37:48,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068581) roll requested 2024-11-11T16:37:48,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068589 2024-11-11T16:37:48,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068589 2024-11-11T16:37:48,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068589) roll requested 2024-11-11T16:37:48,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068597 2024-11-11T16:37:48,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068597 2024-11-11T16:37:48,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068597) roll requested 2024-11-11T16:37:48,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068605 2024-11-11T16:37:48,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068605 2024-11-11T16:37:48,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068605) roll requested 2024-11-11T16:37:48,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068614 2024-11-11T16:37:48,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068614 2024-11-11T16:37:48,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068614) roll requested 2024-11-11T16:37:48,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068621 2024-11-11T16:37:48,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068621 2024-11-11T16:37:48,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068621) roll requested 2024-11-11T16:37:48,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068629 2024-11-11T16:37:48,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068629 2024-11-11T16:37:48,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068629) roll requested 2024-11-11T16:37:48,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068636 2024-11-11T16:37:48,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068636 2024-11-11T16:37:48,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068636) roll requested 2024-11-11T16:37:48,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068643 2024-11-11T16:37:48,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068643 2024-11-11T16:37:48,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068643) roll requested 2024-11-11T16:37:48,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068651 2024-11-11T16:37:48,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068651 2024-11-11T16:37:48,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068651) roll requested 2024-11-11T16:37:48,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068658 2024-11-11T16:37:48,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068658 2024-11-11T16:37:48,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068658) roll requested 2024-11-11T16:37:48,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068665 2024-11-11T16:37:48,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068665 2024-11-11T16:37:48,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068665) roll requested 2024-11-11T16:37:48,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068673 2024-11-11T16:37:48,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068673 2024-11-11T16:37:48,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068673) roll requested 2024-11-11T16:37:48,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068680 2024-11-11T16:37:48,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068680 2024-11-11T16:37:48,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068680) roll requested 2024-11-11T16:37:48,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068690 2024-11-11T16:37:48,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068690 2024-11-11T16:37:48,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068690) roll requested 2024-11-11T16:37:48,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068697 2024-11-11T16:37:48,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068697 2024-11-11T16:37:48,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068697) roll requested 2024-11-11T16:37:48,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068704 2024-11-11T16:37:48,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068704 2024-11-11T16:37:48,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068704) roll requested 2024-11-11T16:37:48,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068711 2024-11-11T16:37:48,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068711 2024-11-11T16:37:48,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068711) roll requested 2024-11-11T16:37:48,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068719 2024-11-11T16:37:48,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068719 2024-11-11T16:37:48,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068719) roll requested 2024-11-11T16:37:48,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068726 2024-11-11T16:37:48,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068726 2024-11-11T16:37:48,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068726) roll requested 2024-11-11T16:37:48,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068733 2024-11-11T16:37:48,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068733 2024-11-11T16:37:48,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068733) roll requested 2024-11-11T16:37:48,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068740 2024-11-11T16:37:48,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068740 2024-11-11T16:37:48,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068740) roll requested 2024-11-11T16:37:48,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068747 2024-11-11T16:37:48,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068747 2024-11-11T16:37:48,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068747) roll requested 2024-11-11T16:37:48,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068754 2024-11-11T16:37:48,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068754 2024-11-11T16:37:48,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068754) roll requested 2024-11-11T16:37:48,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068761 2024-11-11T16:37:48,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068761 2024-11-11T16:37:48,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068761) roll requested 2024-11-11T16:37:48,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068769 2024-11-11T16:37:48,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068769 2024-11-11T16:37:48,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068769) roll requested 2024-11-11T16:37:48,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068776 2024-11-11T16:37:48,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068776 2024-11-11T16:37:48,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068776) roll requested 2024-11-11T16:37:48,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068782 2024-11-11T16:37:48,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068782 2024-11-11T16:37:48,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068782) roll requested 2024-11-11T16:37:48,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068788 2024-11-11T16:37:48,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068788 2024-11-11T16:37:48,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068788) roll requested 2024-11-11T16:37:48,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068795 2024-11-11T16:37:48,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068795 2024-11-11T16:37:48,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068795) roll requested 2024-11-11T16:37:48,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068802 2024-11-11T16:37:48,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068802 2024-11-11T16:37:48,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068802) roll requested 2024-11-11T16:37:48,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068808 2024-11-11T16:37:48,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068808 2024-11-11T16:37:48,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068808) roll requested 2024-11-11T16:37:48,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068815 2024-11-11T16:37:48,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068815 2024-11-11T16:37:48,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068815) roll requested 2024-11-11T16:37:48,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068821 2024-11-11T16:37:48,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068821 2024-11-11T16:37:48,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068821) roll requested 2024-11-11T16:37:48,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068827 2024-11-11T16:37:48,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068827 2024-11-11T16:37:48,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068827) roll requested 2024-11-11T16:37:48,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068834 2024-11-11T16:37:48,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068834 2024-11-11T16:37:48,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068834) roll requested 2024-11-11T16:37:48,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068840 2024-11-11T16:37:48,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068840 2024-11-11T16:37:48,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068840) roll requested 2024-11-11T16:37:48,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068846 2024-11-11T16:37:48,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068846 2024-11-11T16:37:48,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068846) roll requested 2024-11-11T16:37:48,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068853 2024-11-11T16:37:48,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068853 2024-11-11T16:37:48,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068853) roll requested 2024-11-11T16:37:48,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068860 2024-11-11T16:37:48,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068860 2024-11-11T16:37:48,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068860) roll requested 2024-11-11T16:37:48,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068866 2024-11-11T16:37:48,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068866 2024-11-11T16:37:48,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068866) roll requested 2024-11-11T16:37:48,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068873 2024-11-11T16:37:48,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068873 2024-11-11T16:37:48,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068873) roll requested 2024-11-11T16:37:48,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068879 2024-11-11T16:37:48,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068879 2024-11-11T16:37:48,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068879) roll requested 2024-11-11T16:37:48,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068886 2024-11-11T16:37:48,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068886 2024-11-11T16:37:48,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068886) roll requested 2024-11-11T16:37:48,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068893 2024-11-11T16:37:48,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068893 2024-11-11T16:37:48,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068893) roll requested 2024-11-11T16:37:48,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068900 2024-11-11T16:37:48,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068900 2024-11-11T16:37:48,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068900) roll requested 2024-11-11T16:37:48,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068906 2024-11-11T16:37:48,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068906 2024-11-11T16:37:48,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068906) roll requested 2024-11-11T16:37:48,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068913 2024-11-11T16:37:48,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068913 2024-11-11T16:37:48,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068913) roll requested 2024-11-11T16:37:48,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068920 2024-11-11T16:37:48,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068920 2024-11-11T16:37:48,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068920) roll requested 2024-11-11T16:37:48,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068926 2024-11-11T16:37:48,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068926 2024-11-11T16:37:48,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068926) roll requested 2024-11-11T16:37:48,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068933 2024-11-11T16:37:48,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068933 2024-11-11T16:37:48,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068933) roll requested 2024-11-11T16:37:48,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068939 2024-11-11T16:37:48,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068939 2024-11-11T16:37:48,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068939) roll requested 2024-11-11T16:37:48,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068945 2024-11-11T16:37:48,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068945 2024-11-11T16:37:48,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068945) roll requested 2024-11-11T16:37:48,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068952 2024-11-11T16:37:48,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068952 2024-11-11T16:37:48,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068952) roll requested 2024-11-11T16:37:48,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068958 2024-11-11T16:37:48,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068958 2024-11-11T16:37:48,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068958) roll requested 2024-11-11T16:37:48,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068964 2024-11-11T16:37:48,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068964 2024-11-11T16:37:48,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068964) roll requested 2024-11-11T16:37:48,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068971 2024-11-11T16:37:48,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068971 2024-11-11T16:37:48,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068971) roll requested 2024-11-11T16:37:48,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068978 2024-11-11T16:37:48,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068978 2024-11-11T16:37:48,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068978) roll requested 2024-11-11T16:37:48,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068985 2024-11-11T16:37:48,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068985 2024-11-11T16:37:48,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068985) roll requested 2024-11-11T16:37:48,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068993 2024-11-11T16:37:48,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:48,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068993 2024-11-11T16:37:48,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:48,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:48,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068993) roll requested 2024-11-11T16:37:49,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343068999 2024-11-11T16:37:49,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068999 2024-11-11T16:37:49,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343068999) roll requested 2024-11-11T16:37:49,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069006 2024-11-11T16:37:49,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343068999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069006 2024-11-11T16:37:49,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069006) roll requested 2024-11-11T16:37:49,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069013 2024-11-11T16:37:49,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069013 2024-11-11T16:37:49,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069013) roll requested 2024-11-11T16:37:49,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069019 2024-11-11T16:37:49,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069019 2024-11-11T16:37:49,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069019) roll requested 2024-11-11T16:37:49,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069026 2024-11-11T16:37:49,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069026 2024-11-11T16:37:49,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069026) roll requested 2024-11-11T16:37:49,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069032 2024-11-11T16:37:49,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069032 2024-11-11T16:37:49,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069032) roll requested 2024-11-11T16:37:49,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069039 2024-11-11T16:37:49,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069039 2024-11-11T16:37:49,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069039) roll requested 2024-11-11T16:37:49,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069045 2024-11-11T16:37:49,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069045 2024-11-11T16:37:49,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069045) roll requested 2024-11-11T16:37:49,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069052 2024-11-11T16:37:49,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069052 2024-11-11T16:37:49,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069052) roll requested 2024-11-11T16:37:49,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069058 2024-11-11T16:37:49,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069058 2024-11-11T16:37:49,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069058) roll requested 2024-11-11T16:37:49,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069065 2024-11-11T16:37:49,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069065 2024-11-11T16:37:49,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069065) roll requested 2024-11-11T16:37:49,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069071 2024-11-11T16:37:49,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069071 2024-11-11T16:37:49,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069071) roll requested 2024-11-11T16:37:49,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069079 2024-11-11T16:37:49,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069079 2024-11-11T16:37:49,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069079) roll requested 2024-11-11T16:37:49,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069086 2024-11-11T16:37:49,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069086 2024-11-11T16:37:49,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069086) roll requested 2024-11-11T16:37:49,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069092 2024-11-11T16:37:49,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069092 2024-11-11T16:37:49,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069092) roll requested 2024-11-11T16:37:49,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069099 2024-11-11T16:37:49,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069099 2024-11-11T16:37:49,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069099) roll requested 2024-11-11T16:37:49,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069107 2024-11-11T16:37:49,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069107 2024-11-11T16:37:49,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069107) roll requested 2024-11-11T16:37:49,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069113 2024-11-11T16:37:49,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069113 2024-11-11T16:37:49,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069113) roll requested 2024-11-11T16:37:49,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069119 2024-11-11T16:37:49,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069119 2024-11-11T16:37:49,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069119) roll requested 2024-11-11T16:37:49,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069126 2024-11-11T16:37:49,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069126 2024-11-11T16:37:49,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069126) roll requested 2024-11-11T16:37:49,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069133 2024-11-11T16:37:49,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069133 2024-11-11T16:37:49,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069133) roll requested 2024-11-11T16:37:49,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069139 2024-11-11T16:37:49,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069139 2024-11-11T16:37:49,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069139) roll requested 2024-11-11T16:37:49,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069146 2024-11-11T16:37:49,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069146 2024-11-11T16:37:49,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069146) roll requested 2024-11-11T16:37:49,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069152 2024-11-11T16:37:49,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069152 2024-11-11T16:37:49,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069152) roll requested 2024-11-11T16:37:49,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069159 2024-11-11T16:37:49,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069159 2024-11-11T16:37:49,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069159) roll requested 2024-11-11T16:37:49,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069165 2024-11-11T16:37:49,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069165 2024-11-11T16:37:49,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069165) roll requested 2024-11-11T16:37:49,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069172 2024-11-11T16:37:49,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069172 2024-11-11T16:37:49,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069172) roll requested 2024-11-11T16:37:49,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069179 2024-11-11T16:37:49,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069179 2024-11-11T16:37:49,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069179) roll requested 2024-11-11T16:37:49,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069185 2024-11-11T16:37:49,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069185 2024-11-11T16:37:49,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069185) roll requested 2024-11-11T16:37:49,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069191 2024-11-11T16:37:49,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069191 2024-11-11T16:37:49,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069191) roll requested 2024-11-11T16:37:49,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069199 2024-11-11T16:37:49,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069199 2024-11-11T16:37:49,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069199) roll requested 2024-11-11T16:37:49,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069206 2024-11-11T16:37:49,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069206 2024-11-11T16:37:49,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069206) roll requested 2024-11-11T16:37:49,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069212 2024-11-11T16:37:49,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069212 2024-11-11T16:37:49,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069212) roll requested 2024-11-11T16:37:49,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069218 2024-11-11T16:37:49,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069218 2024-11-11T16:37:49,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069218) roll requested 2024-11-11T16:37:49,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069225 2024-11-11T16:37:49,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069225 2024-11-11T16:37:49,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069225) roll requested 2024-11-11T16:37:49,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069231 2024-11-11T16:37:49,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069231 2024-11-11T16:37:49,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069231) roll requested 2024-11-11T16:37:49,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069238 2024-11-11T16:37:49,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069238 2024-11-11T16:37:49,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069238) roll requested 2024-11-11T16:37:49,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069244 2024-11-11T16:37:49,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069244 2024-11-11T16:37:49,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069244) roll requested 2024-11-11T16:37:49,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069251 2024-11-11T16:37:49,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069251 2024-11-11T16:37:49,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069251) roll requested 2024-11-11T16:37:49,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069257 2024-11-11T16:37:49,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069257 2024-11-11T16:37:49,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069257) roll requested 2024-11-11T16:37:49,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069264 2024-11-11T16:37:49,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069264 2024-11-11T16:37:49,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069264) roll requested 2024-11-11T16:37:49,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069270 2024-11-11T16:37:49,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069270 2024-11-11T16:37:49,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069270) roll requested 2024-11-11T16:37:49,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069278 2024-11-11T16:37:49,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069278 2024-11-11T16:37:49,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069278) roll requested 2024-11-11T16:37:49,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069286 2024-11-11T16:37:49,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069286 2024-11-11T16:37:49,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069286) roll requested 2024-11-11T16:37:49,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069292 2024-11-11T16:37:49,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069292 2024-11-11T16:37:49,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069292) roll requested 2024-11-11T16:37:49,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069299 2024-11-11T16:37:49,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069299 2024-11-11T16:37:49,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069299) roll requested 2024-11-11T16:37:49,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069307 2024-11-11T16:37:49,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069307 2024-11-11T16:37:49,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069307) roll requested 2024-11-11T16:37:49,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069317 2024-11-11T16:37:49,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069317 2024-11-11T16:37:49,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069317) roll requested 2024-11-11T16:37:49,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069337 2024-11-11T16:37:49,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069337 2024-11-11T16:37:49,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069337) roll requested 2024-11-11T16:37:49,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069350 2024-11-11T16:37:49,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069350 2024-11-11T16:37:49,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069350) roll requested 2024-11-11T16:37:49,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069361 2024-11-11T16:37:49,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069361 2024-11-11T16:37:49,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069361) roll requested 2024-11-11T16:37:49,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069371 2024-11-11T16:37:49,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069371 2024-11-11T16:37:49,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069371) roll requested 2024-11-11T16:37:49,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069379 2024-11-11T16:37:49,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069379 2024-11-11T16:37:49,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069379) roll requested 2024-11-11T16:37:49,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069387 2024-11-11T16:37:49,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069387 2024-11-11T16:37:49,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069387) roll requested 2024-11-11T16:37:49,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069394 2024-11-11T16:37:49,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069394 2024-11-11T16:37:49,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069394) roll requested 2024-11-11T16:37:49,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069400 2024-11-11T16:37:49,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069400 2024-11-11T16:37:49,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069400) roll requested 2024-11-11T16:37:49,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069407 2024-11-11T16:37:49,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069407 2024-11-11T16:37:49,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069407) roll requested 2024-11-11T16:37:49,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069413 2024-11-11T16:37:49,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069413 2024-11-11T16:37:49,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069413) roll requested 2024-11-11T16:37:49,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069420 2024-11-11T16:37:49,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069420 2024-11-11T16:37:49,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069420) roll requested 2024-11-11T16:37:49,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069426 2024-11-11T16:37:49,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069426 2024-11-11T16:37:49,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069426) roll requested 2024-11-11T16:37:49,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069432 2024-11-11T16:37:49,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069432 2024-11-11T16:37:49,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069432) roll requested 2024-11-11T16:37:49,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069439 2024-11-11T16:37:49,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069439 2024-11-11T16:37:49,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069439) roll requested 2024-11-11T16:37:49,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069445 2024-11-11T16:37:49,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069445 2024-11-11T16:37:49,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069445) roll requested 2024-11-11T16:37:49,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069452 2024-11-11T16:37:49,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069452 2024-11-11T16:37:49,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069452) roll requested 2024-11-11T16:37:49,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069459 2024-11-11T16:37:49,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069459 2024-11-11T16:37:49,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069459) roll requested 2024-11-11T16:37:49,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069466 2024-11-11T16:37:49,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069466 2024-11-11T16:37:49,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069466) roll requested 2024-11-11T16:37:49,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069473 2024-11-11T16:37:49,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069473 2024-11-11T16:37:49,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069473) roll requested 2024-11-11T16:37:49,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069480 2024-11-11T16:37:49,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069480 2024-11-11T16:37:49,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069480) roll requested 2024-11-11T16:37:49,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069496 2024-11-11T16:37:49,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069496 2024-11-11T16:37:49,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069496) roll requested 2024-11-11T16:37:49,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069505 2024-11-11T16:37:49,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069505 2024-11-11T16:37:49,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069505) roll requested 2024-11-11T16:37:49,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069513 2024-11-11T16:37:49,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069513 2024-11-11T16:37:49,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069513) roll requested 2024-11-11T16:37:49,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069519 2024-11-11T16:37:49,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069519 2024-11-11T16:37:49,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069519) roll requested 2024-11-11T16:37:49,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069526 2024-11-11T16:37:49,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069526 2024-11-11T16:37:49,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069526) roll requested 2024-11-11T16:37:49,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069533 2024-11-11T16:37:49,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069533 2024-11-11T16:37:49,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069533) roll requested 2024-11-11T16:37:49,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069539 2024-11-11T16:37:49,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069539 2024-11-11T16:37:49,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069539) roll requested 2024-11-11T16:37:49,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069546 2024-11-11T16:37:49,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069546 2024-11-11T16:37:49,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069546) roll requested 2024-11-11T16:37:49,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069552 2024-11-11T16:37:49,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069552 2024-11-11T16:37:49,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069552) roll requested 2024-11-11T16:37:49,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069559 2024-11-11T16:37:49,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069559 2024-11-11T16:37:49,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069559) roll requested 2024-11-11T16:37:49,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069565 2024-11-11T16:37:49,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069565 2024-11-11T16:37:49,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069565) roll requested 2024-11-11T16:37:49,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069572 2024-11-11T16:37:49,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069572 2024-11-11T16:37:49,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069572) roll requested 2024-11-11T16:37:49,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069578 2024-11-11T16:37:49,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069578 2024-11-11T16:37:49,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069578) roll requested 2024-11-11T16:37:49,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069585 2024-11-11T16:37:49,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069585 2024-11-11T16:37:49,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069585) roll requested 2024-11-11T16:37:49,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069591 2024-11-11T16:37:49,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069591 2024-11-11T16:37:49,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069591) roll requested 2024-11-11T16:37:49,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069597 2024-11-11T16:37:49,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069597 2024-11-11T16:37:49,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069597) roll requested 2024-11-11T16:37:49,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069605 2024-11-11T16:37:49,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069605 2024-11-11T16:37:49,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069605) roll requested 2024-11-11T16:37:49,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069613 2024-11-11T16:37:49,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069613 2024-11-11T16:37:49,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069613) roll requested 2024-11-11T16:37:49,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069621 2024-11-11T16:37:49,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069621 2024-11-11T16:37:49,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069621) roll requested 2024-11-11T16:37:49,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069628 2024-11-11T16:37:49,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069628 2024-11-11T16:37:49,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069628) roll requested 2024-11-11T16:37:49,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069634 2024-11-11T16:37:49,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069634 2024-11-11T16:37:49,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069634) roll requested 2024-11-11T16:37:49,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069641 2024-11-11T16:37:49,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069641 2024-11-11T16:37:49,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069641) roll requested 2024-11-11T16:37:49,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069649 2024-11-11T16:37:49,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069649 2024-11-11T16:37:49,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069649) roll requested 2024-11-11T16:37:49,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069662 2024-11-11T16:37:49,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069662 2024-11-11T16:37:49,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069662) roll requested 2024-11-11T16:37:49,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069670 2024-11-11T16:37:49,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069670 2024-11-11T16:37:49,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069670) roll requested 2024-11-11T16:37:49,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069678 2024-11-11T16:37:49,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069678 2024-11-11T16:37:49,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069678) roll requested 2024-11-11T16:37:49,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069685 2024-11-11T16:37:49,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069685 2024-11-11T16:37:49,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069685) roll requested 2024-11-11T16:37:49,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069692 2024-11-11T16:37:49,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069692 2024-11-11T16:37:49,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069692) roll requested 2024-11-11T16:37:49,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069700 2024-11-11T16:37:49,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069700 2024-11-11T16:37:49,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069700) roll requested 2024-11-11T16:37:49,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069713 2024-11-11T16:37:49,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069713 2024-11-11T16:37:49,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069713) roll requested 2024-11-11T16:37:49,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069732 2024-11-11T16:37:49,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069732 2024-11-11T16:37:49,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069732) roll requested 2024-11-11T16:37:49,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069741 2024-11-11T16:37:49,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069741 2024-11-11T16:37:49,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069741) roll requested 2024-11-11T16:37:49,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069751 2024-11-11T16:37:49,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069751 2024-11-11T16:37:49,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069751) roll requested 2024-11-11T16:37:49,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069769 2024-11-11T16:37:49,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069769 2024-11-11T16:37:49,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069769) roll requested 2024-11-11T16:37:49,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069785 2024-11-11T16:37:49,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069785 2024-11-11T16:37:49,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069785) roll requested 2024-11-11T16:37:49,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069803 2024-11-11T16:37:49,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069803 2024-11-11T16:37:49,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069803) roll requested 2024-11-11T16:37:49,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069829 2024-11-11T16:37:49,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069829 2024-11-11T16:37:49,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069829) roll requested 2024-11-11T16:37:49,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069840 2024-11-11T16:37:49,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069840 2024-11-11T16:37:49,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069840) roll requested 2024-11-11T16:37:49,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069853 2024-11-11T16:37:49,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069853 2024-11-11T16:37:49,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069853) roll requested 2024-11-11T16:37:49,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069864 2024-11-11T16:37:49,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069864 2024-11-11T16:37:49,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069864) roll requested 2024-11-11T16:37:49,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069873 2024-11-11T16:37:49,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069873 2024-11-11T16:37:49,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069873) roll requested 2024-11-11T16:37:49,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069880 2024-11-11T16:37:49,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069880 2024-11-11T16:37:49,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069880) roll requested 2024-11-11T16:37:49,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069887 2024-11-11T16:37:49,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069887 2024-11-11T16:37:49,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069887) roll requested 2024-11-11T16:37:49,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069896 2024-11-11T16:37:49,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069896 2024-11-11T16:37:49,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069896) roll requested 2024-11-11T16:37:49,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069903 2024-11-11T16:37:49,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069903 2024-11-11T16:37:49,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069903) roll requested 2024-11-11T16:37:49,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069910 2024-11-11T16:37:49,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069910 2024-11-11T16:37:49,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069910) roll requested 2024-11-11T16:37:49,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069919 2024-11-11T16:37:49,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069919 2024-11-11T16:37:49,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069919) roll requested 2024-11-11T16:37:49,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069926 2024-11-11T16:37:49,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069926 2024-11-11T16:37:49,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069926) roll requested 2024-11-11T16:37:49,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069933 2024-11-11T16:37:49,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069933 2024-11-11T16:37:49,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069933) roll requested 2024-11-11T16:37:49,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069947 2024-11-11T16:37:49,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069947 2024-11-11T16:37:49,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069947) roll requested 2024-11-11T16:37:49,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069960 2024-11-11T16:37:49,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069960 2024-11-11T16:37:49,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069960) roll requested 2024-11-11T16:37:49,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069969 2024-11-11T16:37:49,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069969 2024-11-11T16:37:49,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069969) roll requested 2024-11-11T16:37:49,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069977 2024-11-11T16:37:49,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069977 2024-11-11T16:37:49,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069977) roll requested 2024-11-11T16:37:49,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069983 2024-11-11T16:37:49,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069983 2024-11-11T16:37:49,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069983) roll requested 2024-11-11T16:37:49,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069990 2024-11-11T16:37:49,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:49,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069990 2024-11-11T16:37:49,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:49,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:49,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069990) roll requested 2024-11-11T16:37:49,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343069997 2024-11-11T16:37:50,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069997 2024-11-11T16:37:50,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343069997) roll requested 2024-11-11T16:37:50,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070004 2024-11-11T16:37:50,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343069997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070004 2024-11-11T16:37:50,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070004) roll requested 2024-11-11T16:37:50,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070010 2024-11-11T16:37:50,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070010 2024-11-11T16:37:50,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070010) roll requested 2024-11-11T16:37:50,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070016 2024-11-11T16:37:50,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070016 2024-11-11T16:37:50,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070016) roll requested 2024-11-11T16:37:50,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070023 2024-11-11T16:37:50,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070023 2024-11-11T16:37:50,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070023) roll requested 2024-11-11T16:37:50,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070029 2024-11-11T16:37:50,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070029 2024-11-11T16:37:50,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070029) roll requested 2024-11-11T16:37:50,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070036 2024-11-11T16:37:50,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070036 2024-11-11T16:37:50,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070036) roll requested 2024-11-11T16:37:50,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070042 2024-11-11T16:37:50,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070042 2024-11-11T16:37:50,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070042) roll requested 2024-11-11T16:37:50,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070049 2024-11-11T16:37:50,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070049 2024-11-11T16:37:50,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070049) roll requested 2024-11-11T16:37:50,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070055 2024-11-11T16:37:50,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070055 2024-11-11T16:37:50,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070055) roll requested 2024-11-11T16:37:50,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070062 2024-11-11T16:37:50,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070062 2024-11-11T16:37:50,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070062) roll requested 2024-11-11T16:37:50,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070068 2024-11-11T16:37:50,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070068 2024-11-11T16:37:50,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070068) roll requested 2024-11-11T16:37:50,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070078 2024-11-11T16:37:50,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070078 2024-11-11T16:37:50,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070078) roll requested 2024-11-11T16:37:50,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070085 2024-11-11T16:37:50,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070085 2024-11-11T16:37:50,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070085) roll requested 2024-11-11T16:37:50,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070093 2024-11-11T16:37:50,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070093 2024-11-11T16:37:50,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070093) roll requested 2024-11-11T16:37:50,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070101 2024-11-11T16:37:50,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070101 2024-11-11T16:37:50,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070101) roll requested 2024-11-11T16:37:50,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070109 2024-11-11T16:37:50,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070109 2024-11-11T16:37:50,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070109) roll requested 2024-11-11T16:37:50,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070117 2024-11-11T16:37:50,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070117 2024-11-11T16:37:50,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070117) roll requested 2024-11-11T16:37:50,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070141 2024-11-11T16:37:50,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070141 2024-11-11T16:37:50,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070141) roll requested 2024-11-11T16:37:50,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070157 2024-11-11T16:37:50,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070157 2024-11-11T16:37:50,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070157) roll requested 2024-11-11T16:37:50,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070165 2024-11-11T16:37:50,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070165 2024-11-11T16:37:50,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070165) roll requested 2024-11-11T16:37:50,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070171 2024-11-11T16:37:50,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070171 2024-11-11T16:37:50,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070171) roll requested 2024-11-11T16:37:50,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070178 2024-11-11T16:37:50,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070178 2024-11-11T16:37:50,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070178) roll requested 2024-11-11T16:37:50,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070185 2024-11-11T16:37:50,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070185 2024-11-11T16:37:50,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070185) roll requested 2024-11-11T16:37:50,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070192 2024-11-11T16:37:50,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070192 2024-11-11T16:37:50,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070192) roll requested 2024-11-11T16:37:50,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070199 2024-11-11T16:37:50,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070199 2024-11-11T16:37:50,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070199) roll requested 2024-11-11T16:37:50,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070209 2024-11-11T16:37:50,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070209 2024-11-11T16:37:50,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070209) roll requested 2024-11-11T16:37:50,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070222 2024-11-11T16:37:50,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070222 2024-11-11T16:37:50,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070222) roll requested 2024-11-11T16:37:50,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070233 2024-11-11T16:37:50,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070233 2024-11-11T16:37:50,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070233) roll requested 2024-11-11T16:37:50,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070243 2024-11-11T16:37:50,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070243 2024-11-11T16:37:50,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070243) roll requested 2024-11-11T16:37:50,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070253 2024-11-11T16:37:50,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070253 2024-11-11T16:37:50,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070253) roll requested 2024-11-11T16:37:50,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070261 2024-11-11T16:37:50,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070261 2024-11-11T16:37:50,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070261) roll requested 2024-11-11T16:37:50,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070269 2024-11-11T16:37:50,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070269 2024-11-11T16:37:50,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070269) roll requested 2024-11-11T16:37:50,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070280 2024-11-11T16:37:50,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070280 2024-11-11T16:37:50,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070280) roll requested 2024-11-11T16:37:50,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070296 2024-11-11T16:37:50,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070296 2024-11-11T16:37:50,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070296) roll requested 2024-11-11T16:37:50,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070309 2024-11-11T16:37:50,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070309 2024-11-11T16:37:50,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070309) roll requested 2024-11-11T16:37:50,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070321 2024-11-11T16:37:50,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070321 2024-11-11T16:37:50,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070321) roll requested 2024-11-11T16:37:50,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070329 2024-11-11T16:37:50,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070329 2024-11-11T16:37:50,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070329) roll requested 2024-11-11T16:37:50,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070341 2024-11-11T16:37:50,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070341 2024-11-11T16:37:50,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070341) roll requested 2024-11-11T16:37:50,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070349 2024-11-11T16:37:50,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070349 2024-11-11T16:37:50,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070349) roll requested 2024-11-11T16:37:50,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070356 2024-11-11T16:37:50,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070356 2024-11-11T16:37:50,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070356) roll requested 2024-11-11T16:37:50,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070364 2024-11-11T16:37:50,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070364 2024-11-11T16:37:50,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070364) roll requested 2024-11-11T16:37:50,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070372 2024-11-11T16:37:50,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070372 2024-11-11T16:37:50,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070372) roll requested 2024-11-11T16:37:50,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070381 2024-11-11T16:37:50,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070381 2024-11-11T16:37:50,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070381) roll requested 2024-11-11T16:37:50,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070389 2024-11-11T16:37:50,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070389 2024-11-11T16:37:50,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070389) roll requested 2024-11-11T16:37:50,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070398 2024-11-11T16:37:50,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070398 2024-11-11T16:37:50,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070398) roll requested 2024-11-11T16:37:50,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070405 2024-11-11T16:37:50,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070405 2024-11-11T16:37:50,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070405) roll requested 2024-11-11T16:37:50,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070417 2024-11-11T16:37:50,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070417 2024-11-11T16:37:50,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070417) roll requested 2024-11-11T16:37:50,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070424 2024-11-11T16:37:50,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070424 2024-11-11T16:37:50,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070424) roll requested 2024-11-11T16:37:50,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070432 2024-11-11T16:37:50,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070432 2024-11-11T16:37:50,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070432) roll requested 2024-11-11T16:37:50,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070440 2024-11-11T16:37:50,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070440 2024-11-11T16:37:50,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070440) roll requested 2024-11-11T16:37:50,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070451 2024-11-11T16:37:50,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070451 2024-11-11T16:37:50,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070451) roll requested 2024-11-11T16:37:50,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070458 2024-11-11T16:37:50,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070458 2024-11-11T16:37:50,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070458) roll requested 2024-11-11T16:37:50,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070465 2024-11-11T16:37:50,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070465 2024-11-11T16:37:50,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070465) roll requested 2024-11-11T16:37:50,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070476 2024-11-11T16:37:50,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070476 2024-11-11T16:37:50,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070476) roll requested 2024-11-11T16:37:50,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070487 2024-11-11T16:37:50,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070487 2024-11-11T16:37:50,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070487) roll requested 2024-11-11T16:37:50,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070505 2024-11-11T16:37:50,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070505 2024-11-11T16:37:50,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070505) roll requested 2024-11-11T16:37:50,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070516 2024-11-11T16:37:50,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070516 2024-11-11T16:37:50,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070516) roll requested 2024-11-11T16:37:50,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070533 2024-11-11T16:37:50,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070533 2024-11-11T16:37:50,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070533) roll requested 2024-11-11T16:37:50,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070541 2024-11-11T16:37:50,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070541 2024-11-11T16:37:50,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070541) roll requested 2024-11-11T16:37:50,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070549 2024-11-11T16:37:50,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070549 2024-11-11T16:37:50,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070549) roll requested 2024-11-11T16:37:50,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070560 2024-11-11T16:37:50,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070560 2024-11-11T16:37:50,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070560) roll requested 2024-11-11T16:37:50,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070568 2024-11-11T16:37:50,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070568 2024-11-11T16:37:50,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070568) roll requested 2024-11-11T16:37:50,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070576 2024-11-11T16:37:50,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070576 2024-11-11T16:37:50,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070576) roll requested 2024-11-11T16:37:50,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070596 2024-11-11T16:37:50,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070596 2024-11-11T16:37:50,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070596) roll requested 2024-11-11T16:37:50,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070613 2024-11-11T16:37:50,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070613 2024-11-11T16:37:50,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070613) roll requested 2024-11-11T16:37:50,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070645 2024-11-11T16:37:50,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070645 2024-11-11T16:37:50,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070645) roll requested 2024-11-11T16:37:50,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070665 2024-11-11T16:37:50,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070665 2024-11-11T16:37:50,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070665) roll requested 2024-11-11T16:37:50,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070693 2024-11-11T16:37:50,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070693 2024-11-11T16:37:50,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070693) roll requested 2024-11-11T16:37:50,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070725 2024-11-11T16:37:50,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070725 2024-11-11T16:37:50,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070725) roll requested 2024-11-11T16:37:50,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070743 2024-11-11T16:37:50,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070743 2024-11-11T16:37:50,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070743) roll requested 2024-11-11T16:37:50,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070752 2024-11-11T16:37:50,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070752 2024-11-11T16:37:50,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070752) roll requested 2024-11-11T16:37:50,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070763 2024-11-11T16:37:50,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070763 2024-11-11T16:37:50,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070763) roll requested 2024-11-11T16:37:50,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070770 2024-11-11T16:37:50,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070770 2024-11-11T16:37:50,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070770) roll requested 2024-11-11T16:37:50,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070778 2024-11-11T16:37:50,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070778 2024-11-11T16:37:50,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070778) roll requested 2024-11-11T16:37:50,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070785 2024-11-11T16:37:50,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070785 2024-11-11T16:37:50,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070785) roll requested 2024-11-11T16:37:50,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070791 2024-11-11T16:37:50,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070791 2024-11-11T16:37:50,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070791) roll requested 2024-11-11T16:37:50,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070798 2024-11-11T16:37:50,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070798 2024-11-11T16:37:50,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070798) roll requested 2024-11-11T16:37:50,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070807 2024-11-11T16:37:50,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070807 2024-11-11T16:37:50,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070807) roll requested 2024-11-11T16:37:50,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070814 2024-11-11T16:37:50,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070814 2024-11-11T16:37:50,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070814) roll requested 2024-11-11T16:37:50,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070821 2024-11-11T16:37:50,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070821 2024-11-11T16:37:50,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070821) roll requested 2024-11-11T16:37:50,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070829 2024-11-11T16:37:50,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070829 2024-11-11T16:37:50,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070829) roll requested 2024-11-11T16:37:50,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070840 2024-11-11T16:37:50,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070840 2024-11-11T16:37:50,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070840) roll requested 2024-11-11T16:37:50,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070849 2024-11-11T16:37:50,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070849 2024-11-11T16:37:50,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070849) roll requested 2024-11-11T16:37:50,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070857 2024-11-11T16:37:50,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070857 2024-11-11T16:37:50,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070857) roll requested 2024-11-11T16:37:50,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070865 2024-11-11T16:37:50,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070865 2024-11-11T16:37:50,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070865) roll requested 2024-11-11T16:37:50,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070874 2024-11-11T16:37:50,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070874 2024-11-11T16:37:50,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070874) roll requested 2024-11-11T16:37:50,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070882 2024-11-11T16:37:50,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070882 2024-11-11T16:37:50,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070882) roll requested 2024-11-11T16:37:50,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070890 2024-11-11T16:37:50,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070890 2024-11-11T16:37:50,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070890) roll requested 2024-11-11T16:37:50,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070899 2024-11-11T16:37:50,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070899 2024-11-11T16:37:50,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070899) roll requested 2024-11-11T16:37:50,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070907 2024-11-11T16:37:50,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070907 2024-11-11T16:37:50,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070907) roll requested 2024-11-11T16:37:50,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070917 2024-11-11T16:37:50,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070917 2024-11-11T16:37:50,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070917) roll requested 2024-11-11T16:37:50,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070925 2024-11-11T16:37:50,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070925 2024-11-11T16:37:50,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070925) roll requested 2024-11-11T16:37:50,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070933 2024-11-11T16:37:50,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070933 2024-11-11T16:37:50,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070933) roll requested 2024-11-11T16:37:50,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070942 2024-11-11T16:37:50,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070942 2024-11-11T16:37:50,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070942) roll requested 2024-11-11T16:37:50,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070953 2024-11-11T16:37:50,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070953 2024-11-11T16:37:50,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070953) roll requested 2024-11-11T16:37:50,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070961 2024-11-11T16:37:50,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070961 2024-11-11T16:37:50,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070961) roll requested 2024-11-11T16:37:50,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070969 2024-11-11T16:37:50,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070969 2024-11-11T16:37:50,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070969) roll requested 2024-11-11T16:37:50,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070977 2024-11-11T16:37:50,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070977 2024-11-11T16:37:50,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070977) roll requested 2024-11-11T16:37:50,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070985 2024-11-11T16:37:50,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:50,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070985 2024-11-11T16:37:50,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:50,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:50,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070985) roll requested 2024-11-11T16:37:50,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343070994 2024-11-11T16:37:51,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070994 2024-11-11T16:37:51,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343070994) roll requested 2024-11-11T16:37:51,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071004 2024-11-11T16:37:51,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343070994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071004 2024-11-11T16:37:51,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071004) roll requested 2024-11-11T16:37:51,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071012 2024-11-11T16:37:51,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071012 2024-11-11T16:37:51,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071012) roll requested 2024-11-11T16:37:51,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071020 2024-11-11T16:37:51,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071020 2024-11-11T16:37:51,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071020) roll requested 2024-11-11T16:37:51,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071027 2024-11-11T16:37:51,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071027 2024-11-11T16:37:51,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071027) roll requested 2024-11-11T16:37:51,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071034 2024-11-11T16:37:51,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071034 2024-11-11T16:37:51,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071034) roll requested 2024-11-11T16:37:51,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071041 2024-11-11T16:37:51,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071041 2024-11-11T16:37:51,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071041) roll requested 2024-11-11T16:37:51,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071049 2024-11-11T16:37:51,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071049 2024-11-11T16:37:51,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071049) roll requested 2024-11-11T16:37:51,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071062 2024-11-11T16:37:51,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071062 2024-11-11T16:37:51,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071062) roll requested 2024-11-11T16:37:51,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071081 2024-11-11T16:37:51,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071081 2024-11-11T16:37:51,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071081) roll requested 2024-11-11T16:37:51,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071090 2024-11-11T16:37:51,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071090 2024-11-11T16:37:51,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071090) roll requested 2024-11-11T16:37:51,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071099 2024-11-11T16:37:51,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071099 2024-11-11T16:37:51,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071099) roll requested 2024-11-11T16:37:51,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071109 2024-11-11T16:37:51,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071109 2024-11-11T16:37:51,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071109) roll requested 2024-11-11T16:37:51,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071116 2024-11-11T16:37:51,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071116 2024-11-11T16:37:51,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071116) roll requested 2024-11-11T16:37:51,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071125 2024-11-11T16:37:51,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071125 2024-11-11T16:37:51,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071125) roll requested 2024-11-11T16:37:51,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071132 2024-11-11T16:37:51,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071132 2024-11-11T16:37:51,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071132) roll requested 2024-11-11T16:37:51,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071138 2024-11-11T16:37:51,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071138 2024-11-11T16:37:51,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071138) roll requested 2024-11-11T16:37:51,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071145 2024-11-11T16:37:51,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071145 2024-11-11T16:37:51,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071145) roll requested 2024-11-11T16:37:51,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071151 2024-11-11T16:37:51,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071151 2024-11-11T16:37:51,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071151) roll requested 2024-11-11T16:37:51,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071157 2024-11-11T16:37:51,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071157 2024-11-11T16:37:51,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071157) roll requested 2024-11-11T16:37:51,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071163 2024-11-11T16:37:51,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071163 2024-11-11T16:37:51,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071163) roll requested 2024-11-11T16:37:51,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071169 2024-11-11T16:37:51,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071169 2024-11-11T16:37:51,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071169) roll requested 2024-11-11T16:37:51,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071176 2024-11-11T16:37:51,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071176 2024-11-11T16:37:51,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071176) roll requested 2024-11-11T16:37:51,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071182 2024-11-11T16:37:51,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071182 2024-11-11T16:37:51,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071182) roll requested 2024-11-11T16:37:51,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071188 2024-11-11T16:37:51,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071188 2024-11-11T16:37:51,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071188) roll requested 2024-11-11T16:37:51,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071194 2024-11-11T16:37:51,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071194 2024-11-11T16:37:51,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071194) roll requested 2024-11-11T16:37:51,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071201 2024-11-11T16:37:51,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071201 2024-11-11T16:37:51,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071201) roll requested 2024-11-11T16:37:51,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071207 2024-11-11T16:37:51,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071207 2024-11-11T16:37:51,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071207) roll requested 2024-11-11T16:37:51,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071214 2024-11-11T16:37:51,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071214 2024-11-11T16:37:51,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071214) roll requested 2024-11-11T16:37:51,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071221 2024-11-11T16:37:51,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071221 2024-11-11T16:37:51,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071221) roll requested 2024-11-11T16:37:51,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071227 2024-11-11T16:37:51,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071227 2024-11-11T16:37:51,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071227) roll requested 2024-11-11T16:37:51,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071233 2024-11-11T16:37:51,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071233 2024-11-11T16:37:51,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071233) roll requested 2024-11-11T16:37:51,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071239 2024-11-11T16:37:51,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071239 2024-11-11T16:37:51,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071239) roll requested 2024-11-11T16:37:51,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071246 2024-11-11T16:37:51,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071246 2024-11-11T16:37:51,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071246) roll requested 2024-11-11T16:37:51,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071252 2024-11-11T16:37:51,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071252 2024-11-11T16:37:51,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071252) roll requested 2024-11-11T16:37:51,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071258 2024-11-11T16:37:51,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071258 2024-11-11T16:37:51,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071258) roll requested 2024-11-11T16:37:51,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071265 2024-11-11T16:37:51,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071265 2024-11-11T16:37:51,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071265) roll requested 2024-11-11T16:37:51,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071271 2024-11-11T16:37:51,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071271 2024-11-11T16:37:51,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071271) roll requested 2024-11-11T16:37:51,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071278 2024-11-11T16:37:51,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071278 2024-11-11T16:37:51,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071278) roll requested 2024-11-11T16:37:51,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071285 2024-11-11T16:37:51,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071285 2024-11-11T16:37:51,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071285) roll requested 2024-11-11T16:37:51,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071292 2024-11-11T16:37:51,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071292 2024-11-11T16:37:51,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071292) roll requested 2024-11-11T16:37:51,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071304 2024-11-11T16:37:51,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071304 2024-11-11T16:37:51,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071304) roll requested 2024-11-11T16:37:51,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071312 2024-11-11T16:37:51,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071312 2024-11-11T16:37:51,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071312) roll requested 2024-11-11T16:37:51,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071319 2024-11-11T16:37:51,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071319 2024-11-11T16:37:51,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071319) roll requested 2024-11-11T16:37:51,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071325 2024-11-11T16:37:51,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071325 2024-11-11T16:37:51,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071325) roll requested 2024-11-11T16:37:51,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071331 2024-11-11T16:37:51,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071331 2024-11-11T16:37:51,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071331) roll requested 2024-11-11T16:37:51,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071337 2024-11-11T16:37:51,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071337 2024-11-11T16:37:51,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071337) roll requested 2024-11-11T16:37:51,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071344 2024-11-11T16:37:51,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071344 2024-11-11T16:37:51,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071344) roll requested 2024-11-11T16:37:51,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071350 2024-11-11T16:37:51,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071350 2024-11-11T16:37:51,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071350) roll requested 2024-11-11T16:37:51,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071356 2024-11-11T16:37:51,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071356 2024-11-11T16:37:51,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071356) roll requested 2024-11-11T16:37:51,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071363 2024-11-11T16:37:51,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071363 2024-11-11T16:37:51,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071363) roll requested 2024-11-11T16:37:51,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071369 2024-11-11T16:37:51,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071369 2024-11-11T16:37:51,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071369) roll requested 2024-11-11T16:37:51,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071375 2024-11-11T16:37:51,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071375 2024-11-11T16:37:51,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071375) roll requested 2024-11-11T16:37:51,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071382 2024-11-11T16:37:51,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071382 2024-11-11T16:37:51,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071382) roll requested 2024-11-11T16:37:51,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071388 2024-11-11T16:37:51,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071388 2024-11-11T16:37:51,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071388) roll requested 2024-11-11T16:37:51,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071395 2024-11-11T16:37:51,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071395 2024-11-11T16:37:51,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071395) roll requested 2024-11-11T16:37:51,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071401 2024-11-11T16:37:51,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071401 2024-11-11T16:37:51,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071401) roll requested 2024-11-11T16:37:51,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071408 2024-11-11T16:37:51,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071408 2024-11-11T16:37:51,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071408) roll requested 2024-11-11T16:37:51,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071415 2024-11-11T16:37:51,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071415 2024-11-11T16:37:51,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071415) roll requested 2024-11-11T16:37:51,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071421 2024-11-11T16:37:51,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071421 2024-11-11T16:37:51,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071421) roll requested 2024-11-11T16:37:51,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071427 2024-11-11T16:37:51,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071427 2024-11-11T16:37:51,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071427) roll requested 2024-11-11T16:37:51,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071437 2024-11-11T16:37:51,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071437 2024-11-11T16:37:51,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071437) roll requested 2024-11-11T16:37:51,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071443 2024-11-11T16:37:51,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071443 2024-11-11T16:37:51,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071443) roll requested 2024-11-11T16:37:51,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071450 2024-11-11T16:37:51,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071450 2024-11-11T16:37:51,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071450) roll requested 2024-11-11T16:37:51,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071456 2024-11-11T16:37:51,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071456 2024-11-11T16:37:51,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071456) roll requested 2024-11-11T16:37:51,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071462 2024-11-11T16:37:51,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071462 2024-11-11T16:37:51,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071462) roll requested 2024-11-11T16:37:51,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071469 2024-11-11T16:37:51,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071469 2024-11-11T16:37:51,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071469) roll requested 2024-11-11T16:37:51,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071475 2024-11-11T16:37:51,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071475 2024-11-11T16:37:51,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071475) roll requested 2024-11-11T16:37:51,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071482 2024-11-11T16:37:51,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071482 2024-11-11T16:37:51,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071482) roll requested 2024-11-11T16:37:51,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071488 2024-11-11T16:37:51,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071488 2024-11-11T16:37:51,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071488) roll requested 2024-11-11T16:37:51,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071494 2024-11-11T16:37:51,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071494 2024-11-11T16:37:51,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071494) roll requested 2024-11-11T16:37:51,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071501 2024-11-11T16:37:51,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071501 2024-11-11T16:37:51,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071501) roll requested 2024-11-11T16:37:51,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071507 2024-11-11T16:37:51,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071507 2024-11-11T16:37:51,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071507) roll requested 2024-11-11T16:37:51,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071513 2024-11-11T16:37:51,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071513 2024-11-11T16:37:51,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071513) roll requested 2024-11-11T16:37:51,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071519 2024-11-11T16:37:51,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071519 2024-11-11T16:37:51,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071519) roll requested 2024-11-11T16:37:51,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071526 2024-11-11T16:37:51,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071526 2024-11-11T16:37:51,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071526) roll requested 2024-11-11T16:37:51,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071533 2024-11-11T16:37:51,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071533 2024-11-11T16:37:51,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:37:51,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071533) roll requested 2024-11-11T16:37:51,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071540 2024-11-11T16:37:51,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071540 2024-11-11T16:37:51,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,553 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071540) roll requested 2024-11-11T16:37:51,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071554 2024-11-11T16:37:51,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071540 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071554 2024-11-11T16:37:51,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071554) roll requested 2024-11-11T16:37:51,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071561 2024-11-11T16:37:51,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071554 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071561 2024-11-11T16:37:51,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,569 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071561) roll requested 2024-11-11T16:37:51,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071569 2024-11-11T16:37:51,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071561 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071569 2024-11-11T16:37:51,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071569) roll requested 2024-11-11T16:37:51,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071577 2024-11-11T16:37:51,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071577 2024-11-11T16:37:51,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,585 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071577) roll requested 2024-11-11T16:37:51,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071585 2024-11-11T16:37:51,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071585 2024-11-11T16:37:51,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071585) roll requested 2024-11-11T16:37:51,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071593 2024-11-11T16:37:51,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071593 2024-11-11T16:37:51,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,609 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071593) roll requested 2024-11-11T16:37:51,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071609 2024-11-11T16:37:51,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071609 2024-11-11T16:37:51,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,622 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071609) roll requested 2024-11-11T16:37:51,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071622 2024-11-11T16:37:51,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071622 2024-11-11T16:37:51,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071622) roll requested 2024-11-11T16:37:51,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071630 2024-11-11T16:37:51,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071622 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071630 2024-11-11T16:37:51,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071630) roll requested 2024-11-11T16:37:51,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071637 2024-11-11T16:37:51,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071637 2024-11-11T16:37:51,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,643 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071637) roll requested 2024-11-11T16:37:51,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071644 2024-11-11T16:37:51,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071644 2024-11-11T16:37:51,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071644) roll requested 2024-11-11T16:37:51,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071651 2024-11-11T16:37:51,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071644 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071651 2024-11-11T16:37:51,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,657 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071651) roll requested 2024-11-11T16:37:51,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071658 2024-11-11T16:37:51,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071658 2024-11-11T16:37:51,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071658) roll requested 2024-11-11T16:37:51,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071665 2024-11-11T16:37:51,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071665 2024-11-11T16:37:51,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,671 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071665) roll requested 2024-11-11T16:37:51,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071672 2024-11-11T16:37:51,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071672 2024-11-11T16:37:51,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,678 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071672) roll requested 2024-11-11T16:37:51,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071678 2024-11-11T16:37:51,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071672 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071678 2024-11-11T16:37:51,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,685 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071678) roll requested 2024-11-11T16:37:51,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071685 2024-11-11T16:37:51,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071685 2024-11-11T16:37:51,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,691 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071685) roll requested 2024-11-11T16:37:51,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071691 2024-11-11T16:37:51,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071691 2024-11-11T16:37:51,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071691) roll requested 2024-11-11T16:37:51,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071698 2024-11-11T16:37:51,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071698 2024-11-11T16:37:51,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,704 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071698) roll requested 2024-11-11T16:37:51,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071704 2024-11-11T16:37:51,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071698 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071704 2024-11-11T16:37:51,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,711 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071704) roll requested 2024-11-11T16:37:51,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071711 2024-11-11T16:37:51,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071704 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071711 2024-11-11T16:37:51,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,718 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071711) roll requested 2024-11-11T16:37:51,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071718 2024-11-11T16:37:51,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071711 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071718 2024-11-11T16:37:51,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071718) roll requested 2024-11-11T16:37:51,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071729 2024-11-11T16:37:51,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071718 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071729 2024-11-11T16:37:51,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,742 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071729) roll requested 2024-11-11T16:37:51,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071742 2024-11-11T16:37:51,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071729 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071742 2024-11-11T16:37:51,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,750 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071742) roll requested 2024-11-11T16:37:51,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071750 2024-11-11T16:37:51,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071742 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071750 2024-11-11T16:37:51,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071750) roll requested 2024-11-11T16:37:51,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071758 2024-11-11T16:37:51,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071750 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071758 2024-11-11T16:37:51,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071758) roll requested 2024-11-11T16:37:51,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071767 2024-11-11T16:37:51,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071758 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071767 2024-11-11T16:37:51,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071767) roll requested 2024-11-11T16:37:51,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071774 2024-11-11T16:37:51,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071774 2024-11-11T16:37:51,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,780 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071774) roll requested 2024-11-11T16:37:51,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071780 2024-11-11T16:37:51,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071774 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071780 2024-11-11T16:37:51,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,786 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071780) roll requested 2024-11-11T16:37:51,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071787 2024-11-11T16:37:51,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071780 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071787 2024-11-11T16:37:51,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,794 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071787) roll requested 2024-11-11T16:37:51,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071794 2024-11-11T16:37:51,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071787 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071794 2024-11-11T16:37:51,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071794) roll requested 2024-11-11T16:37:51,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071805 2024-11-11T16:37:51,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071805 2024-11-11T16:37:51,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,813 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071805) roll requested 2024-11-11T16:37:51,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071813 2024-11-11T16:37:51,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071813 2024-11-11T16:37:51,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071813) roll requested 2024-11-11T16:37:51,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071820 2024-11-11T16:37:51,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071820 2024-11-11T16:37:51,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071820) roll requested 2024-11-11T16:37:51,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071827 2024-11-11T16:37:51,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071820 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071827 2024-11-11T16:37:51,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,834 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071827) roll requested 2024-11-11T16:37:51,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071834 2024-11-11T16:37:51,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071827 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071834 2024-11-11T16:37:51,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071834) roll requested 2024-11-11T16:37:51,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071841 2024-11-11T16:37:51,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071834 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071841 2024-11-11T16:37:51,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071841) roll requested 2024-11-11T16:37:51,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071848 2024-11-11T16:37:51,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071841 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071848 2024-11-11T16:37:51,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071848) roll requested 2024-11-11T16:37:51,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071854 2024-11-11T16:37:51,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071848 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071854 2024-11-11T16:37:51,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,861 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071854) roll requested 2024-11-11T16:37:51,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071861 2024-11-11T16:37:51,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071861 2024-11-11T16:37:51,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,868 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071861) roll requested 2024-11-11T16:37:51,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071869 2024-11-11T16:37:51,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071869 2024-11-11T16:37:51,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,876 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071869) roll requested 2024-11-11T16:37:51,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071876 2024-11-11T16:37:51,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071869 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071876 2024-11-11T16:37:51,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,883 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071876) roll requested 2024-11-11T16:37:51,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071883 2024-11-11T16:37:51,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071876 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071883 2024-11-11T16:37:51,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071883) roll requested 2024-11-11T16:37:51,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071890 2024-11-11T16:37:51,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071883 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071890 2024-11-11T16:37:51,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,896 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071890) roll requested 2024-11-11T16:37:51,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071896 2024-11-11T16:37:51,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071896 2024-11-11T16:37:51,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,903 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071896) roll requested 2024-11-11T16:37:51,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071903 2024-11-11T16:37:51,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071896 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071903 2024-11-11T16:37:51,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,910 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071903) roll requested 2024-11-11T16:37:51,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071910 2024-11-11T16:37:51,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071903 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071910 2024-11-11T16:37:51,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071910) roll requested 2024-11-11T16:37:51,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071917 2024-11-11T16:37:51,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071910 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071917 2024-11-11T16:37:51,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071917) roll requested 2024-11-11T16:37:51,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071925 2024-11-11T16:37:51,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071925 2024-11-11T16:37:51,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,931 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071925) roll requested 2024-11-11T16:37:51,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071932 2024-11-11T16:37:51,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071932 2024-11-11T16:37:51,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071932) roll requested 2024-11-11T16:37:51,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071938 2024-11-11T16:37:51,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071932 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071938 2024-11-11T16:37:51,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071938) roll requested 2024-11-11T16:37:51,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071945 2024-11-11T16:37:51,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071945 2024-11-11T16:37:51,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071945) roll requested 2024-11-11T16:37:51,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071952 2024-11-11T16:37:51,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071945 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071952 2024-11-11T16:37:51,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071952) roll requested 2024-11-11T16:37:51,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071959 2024-11-11T16:37:51,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071959 2024-11-11T16:37:51,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071959) roll requested 2024-11-11T16:37:51,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071967 2024-11-11T16:37:51,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071967 2024-11-11T16:37:51,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,975 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,976 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=56, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:51,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:51,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071967) roll requested 2024-11-11T16:37:51,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071977 2024-11-11T16:37:51,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071977 2024-11-11T16:37:51,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,985 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=57, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:51,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,985 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:51,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071977) roll requested 2024-11-11T16:37:51,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071985 2024-11-11T16:37:51,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:51,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071985 2024-11-11T16:37:51,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:51,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=58, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:51,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:51,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:51,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071985) roll requested 2024-11-11T16:37:51,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343071993 2024-11-11T16:37:52,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071993 2024-11-11T16:37:52,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,002 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=59, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,002 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,002 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343071993) roll requested 2024-11-11T16:37:52,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072002 2024-11-11T16:37:52,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343071993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072002 2024-11-11T16:37:52,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=60, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,010 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,010 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072002) roll requested 2024-11-11T16:37:52,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072010 2024-11-11T16:37:52,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072002 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072010 2024-11-11T16:37:52,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,017 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=61, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,017 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,017 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072010) roll requested 2024-11-11T16:37:52,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072017 2024-11-11T16:37:52,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072010 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072017 2024-11-11T16:37:52,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,024 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=62, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072017) roll requested 2024-11-11T16:37:52,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072025 2024-11-11T16:37:52,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072025 2024-11-11T16:37:52,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,038 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=63, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,038 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,038 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072025) roll requested 2024-11-11T16:37:52,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072038 2024-11-11T16:37:52,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072038 2024-11-11T16:37:52,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,050 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=64, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,050 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,050 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072038) roll requested 2024-11-11T16:37:52,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072050 2024-11-11T16:37:52,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072050 2024-11-11T16:37:52,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,062 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=65, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,062 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,062 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072050) roll requested 2024-11-11T16:37:52,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072062 2024-11-11T16:37:52,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072062 2024-11-11T16:37:52,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=66, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072062) roll requested 2024-11-11T16:37:52,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072070 2024-11-11T16:37:52,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072062 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072070 2024-11-11T16:37:52,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=67, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072070) roll requested 2024-11-11T16:37:52,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072078 2024-11-11T16:37:52,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072078 2024-11-11T16:37:52,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,085 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=68, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,085 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072078) roll requested 2024-11-11T16:37:52,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072086 2024-11-11T16:37:52,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072086 2024-11-11T16:37:52,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,102 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=69, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,102 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072086) roll requested 2024-11-11T16:37:52,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072105 2024-11-11T16:37:52,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072105 2024-11-11T16:37:52,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,125 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=70, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,125 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072105) roll requested 2024-11-11T16:37:52,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072125 2024-11-11T16:37:52,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072125 2024-11-11T16:37:52,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=71, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,137 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,137 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072125) roll requested 2024-11-11T16:37:52,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072137 2024-11-11T16:37:52,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072137 2024-11-11T16:37:52,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,164 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=72, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072137) roll requested 2024-11-11T16:37:52,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072165 2024-11-11T16:37:52,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072137 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072165 2024-11-11T16:37:52,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,177 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=73, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072165) roll requested 2024-11-11T16:37:52,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072177 2024-11-11T16:37:52,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072177 2024-11-11T16:37:52,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=74, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072177) roll requested 2024-11-11T16:37:52,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072188 2024-11-11T16:37:52,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072188 2024-11-11T16:37:52,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=75, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072188) roll requested 2024-11-11T16:37:52,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072196 2024-11-11T16:37:52,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072188 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072196 2024-11-11T16:37:52,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,202 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=76, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072196) roll requested 2024-11-11T16:37:52,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072203 2024-11-11T16:37:52,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072203 2024-11-11T16:37:52,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=77, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072203) roll requested 2024-11-11T16:37:52,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072210 2024-11-11T16:37:52,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072210 2024-11-11T16:37:52,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,216 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=78, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,216 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072210) roll requested 2024-11-11T16:37:52,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072216 2024-11-11T16:37:52,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072216 2024-11-11T16:37:52,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,222 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=79, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,222 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,222 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072216) roll requested 2024-11-11T16:37:52,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072222 2024-11-11T16:37:52,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072216 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072222 2024-11-11T16:37:52,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,231 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=80, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,231 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,231 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072222) roll requested 2024-11-11T16:37:52,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072231 2024-11-11T16:37:52,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072222 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072231 2024-11-11T16:37:52,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,241 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=81, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,241 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072231) roll requested 2024-11-11T16:37:52,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072241 2024-11-11T16:37:52,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072231 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072241 2024-11-11T16:37:52,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,248 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=82, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,248 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,249 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072241) roll requested 2024-11-11T16:37:52,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072249 2024-11-11T16:37:52,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072249 2024-11-11T16:37:52,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=83, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072249) roll requested 2024-11-11T16:37:52,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072255 2024-11-11T16:37:52,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072249 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072255 2024-11-11T16:37:52,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,262 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=84, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,262 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,262 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072255) roll requested 2024-11-11T16:37:52,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072262 2024-11-11T16:37:52,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072262 2024-11-11T16:37:52,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=85, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072262) roll requested 2024-11-11T16:37:52,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072269 2024-11-11T16:37:52,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072262 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072269 2024-11-11T16:37:52,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=86, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,276 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072269) roll requested 2024-11-11T16:37:52,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072277 2024-11-11T16:37:52,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072277 2024-11-11T16:37:52,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,283 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=87, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,284 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072277) roll requested 2024-11-11T16:37:52,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072284 2024-11-11T16:37:52,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072284 2024-11-11T16:37:52,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,291 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=88, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072284) roll requested 2024-11-11T16:37:52,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072292 2024-11-11T16:37:52,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072284 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072292 2024-11-11T16:37:52,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,298 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=89, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,298 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,298 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072292) roll requested 2024-11-11T16:37:52,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072298 2024-11-11T16:37:52,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072298 2024-11-11T16:37:52,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=90, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,308 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072298) roll requested 2024-11-11T16:37:52,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072308 2024-11-11T16:37:52,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072298 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072308 2024-11-11T16:37:52,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,315 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=91, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,315 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,315 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072308) roll requested 2024-11-11T16:37:52,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072315 2024-11-11T16:37:52,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072308 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072315 2024-11-11T16:37:52,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,322 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=92, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,322 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,322 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072315) roll requested 2024-11-11T16:37:52,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072322 2024-11-11T16:37:52,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072315 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072322 2024-11-11T16:37:52,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,332 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=93, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,332 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072322) roll requested 2024-11-11T16:37:52,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072332 2024-11-11T16:37:52,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072322 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072332 2024-11-11T16:37:52,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,341 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=94, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,341 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,341 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072332) roll requested 2024-11-11T16:37:52,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072341 2024-11-11T16:37:52,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072332 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072341 2024-11-11T16:37:52,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=95, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072341) roll requested 2024-11-11T16:37:52,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072349 2024-11-11T16:37:52,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072349 2024-11-11T16:37:52,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,357 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=96, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,357 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072349) roll requested 2024-11-11T16:37:52,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072357 2024-11-11T16:37:52,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072357 2024-11-11T16:37:52,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,378 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=97, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,378 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,379 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072357) roll requested 2024-11-11T16:37:52,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072379 2024-11-11T16:37:52,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072379 2024-11-11T16:37:52,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,386 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=98, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,387 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072379) roll requested 2024-11-11T16:37:52,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072387 2024-11-11T16:37:52,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072379 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072387 2024-11-11T16:37:52,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,394 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=99, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,394 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072387) roll requested 2024-11-11T16:37:52,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072394 2024-11-11T16:37:52,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072394 2024-11-11T16:37:52,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=100, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,401 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072394) roll requested 2024-11-11T16:37:52,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072401 2024-11-11T16:37:52,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072401 2024-11-11T16:37:52,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,407 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=101, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,407 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,407 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072401) roll requested 2024-11-11T16:37:52,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072407 2024-11-11T16:37:52,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072407 2024-11-11T16:37:52,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=102, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,414 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072407) roll requested 2024-11-11T16:37:52,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072414 2024-11-11T16:37:52,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072414 2024-11-11T16:37:52,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,421 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=103, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,421 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,421 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072414) roll requested 2024-11-11T16:37:52,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072421 2024-11-11T16:37:52,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072414 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072421 2024-11-11T16:37:52,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=104, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072421) roll requested 2024-11-11T16:37:52,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072428 2024-11-11T16:37:52,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072428 2024-11-11T16:37:52,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,435 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=105, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,435 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,435 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072428) roll requested 2024-11-11T16:37:52,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072435 2024-11-11T16:37:52,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072435 2024-11-11T16:37:52,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,441 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=106, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,441 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072435) roll requested 2024-11-11T16:37:52,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072441 2024-11-11T16:37:52,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072441 2024-11-11T16:37:52,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=107, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,448 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072441) roll requested 2024-11-11T16:37:52,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072448 2024-11-11T16:37:52,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072441 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072448 2024-11-11T16:37:52,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,454 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=108, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,454 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,454 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072448) roll requested 2024-11-11T16:37:52,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072454 2024-11-11T16:37:52,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072448 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072454 2024-11-11T16:37:52,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,461 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=109, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,461 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072454) roll requested 2024-11-11T16:37:52,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072461 2024-11-11T16:37:52,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072454 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072461 2024-11-11T16:37:52,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,468 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=110, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,468 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,468 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072461) roll requested 2024-11-11T16:37:52,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072468 2024-11-11T16:37:52,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072468 2024-11-11T16:37:52,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,474 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=111, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,474 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072468) roll requested 2024-11-11T16:37:52,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072474 2024-11-11T16:37:52,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072468 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072474 2024-11-11T16:37:52,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,480 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=112, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,480 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,481 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072474) roll requested 2024-11-11T16:37:52,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072481 2024-11-11T16:37:52,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072481 2024-11-11T16:37:52,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,487 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=113, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,487 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,487 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072481) roll requested 2024-11-11T16:37:52,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072488 2024-11-11T16:37:52,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072481 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072488 2024-11-11T16:37:52,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,494 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=114, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,494 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,494 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072488) roll requested 2024-11-11T16:37:52,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072494 2024-11-11T16:37:52,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072494 2024-11-11T16:37:52,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,501 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=115, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072494) roll requested 2024-11-11T16:37:52,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072501 2024-11-11T16:37:52,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072494 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072501 2024-11-11T16:37:52,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,507 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=116, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,507 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,507 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072501) roll requested 2024-11-11T16:37:52,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072507 2024-11-11T16:37:52,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072507 2024-11-11T16:37:52,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,515 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=117, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,515 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,516 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072507) roll requested 2024-11-11T16:37:52,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072516 2024-11-11T16:37:52,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072507 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072516 2024-11-11T16:37:52,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=118, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072516) roll requested 2024-11-11T16:37:52,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072523 2024-11-11T16:37:52,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072516 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072523 2024-11-11T16:37:52,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,531 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=119, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,531 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072523) roll requested 2024-11-11T16:37:52,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072531 2024-11-11T16:37:52,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072531 2024-11-11T16:37:52,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,540 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=120, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,540 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,540 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072531) roll requested 2024-11-11T16:37:52,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072540 2024-11-11T16:37:52,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072540 2024-11-11T16:37:52,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=121, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,548 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072540) roll requested 2024-11-11T16:37:52,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072548 2024-11-11T16:37:52,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072540 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072548 2024-11-11T16:37:52,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,556 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=122, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072548) roll requested 2024-11-11T16:37:52,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072557 2024-11-11T16:37:52,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072557 2024-11-11T16:37:52,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=123, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072557) roll requested 2024-11-11T16:37:52,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072573 2024-11-11T16:37:52,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072573 2024-11-11T16:37:52,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,586 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=124, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,586 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,586 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072573) roll requested 2024-11-11T16:37:52,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072586 2024-11-11T16:37:52,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072586 2024-11-11T16:37:52,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,594 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=125, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,595 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072586) roll requested 2024-11-11T16:37:52,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072599 2024-11-11T16:37:52,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072586 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072599 2024-11-11T16:37:52,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,611 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=126, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,612 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072599) roll requested 2024-11-11T16:37:52,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072612 2024-11-11T16:37:52,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072612 2024-11-11T16:37:52,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=127, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072612) roll requested 2024-11-11T16:37:52,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072625 2024-11-11T16:37:52,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072612 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072625 2024-11-11T16:37:52,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=128, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072625) roll requested 2024-11-11T16:37:52,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072633 2024-11-11T16:37:52,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072633 2024-11-11T16:37:52,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=129, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,642 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,642 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072633) roll requested 2024-11-11T16:37:52,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072642 2024-11-11T16:37:52,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072642 2024-11-11T16:37:52,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,649 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,649 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=130, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,649 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072642) roll requested 2024-11-11T16:37:52,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072649 2024-11-11T16:37:52,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072642 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072649 2024-11-11T16:37:52,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,656 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=131, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,656 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072649) roll requested 2024-11-11T16:37:52,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072656 2024-11-11T16:37:52,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072649 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072656 2024-11-11T16:37:52,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,662 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=132, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,663 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072656) roll requested 2024-11-11T16:37:52,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072663 2024-11-11T16:37:52,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072656 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072663 2024-11-11T16:37:52,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,670 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=133, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,670 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,670 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072663) roll requested 2024-11-11T16:37:52,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072670 2024-11-11T16:37:52,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072670 2024-11-11T16:37:52,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=134, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,676 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072670) roll requested 2024-11-11T16:37:52,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072676 2024-11-11T16:37:52,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072676 2024-11-11T16:37:52,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,683 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=135, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,683 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,683 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072676) roll requested 2024-11-11T16:37:52,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072683 2024-11-11T16:37:52,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072676 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072683 2024-11-11T16:37:52,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=136, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072683) roll requested 2024-11-11T16:37:52,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072689 2024-11-11T16:37:52,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072683 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072689 2024-11-11T16:37:52,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=137, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072689) roll requested 2024-11-11T16:37:52,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072696 2024-11-11T16:37:52,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072696 2024-11-11T16:37:52,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,702 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,702 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=138, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072696) roll requested 2024-11-11T16:37:52,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072703 2024-11-11T16:37:52,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072703 2024-11-11T16:37:52,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,713 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,713 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=139, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,713 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072703) roll requested 2024-11-11T16:37:52,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072713 2024-11-11T16:37:52,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072713 2024-11-11T16:37:52,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,720 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=140, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,720 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,720 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072713) roll requested 2024-11-11T16:37:52,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072720 2024-11-11T16:37:52,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072713 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072720 2024-11-11T16:37:52,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=141, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072720) roll requested 2024-11-11T16:37:52,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072727 2024-11-11T16:37:52,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072720 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072727 2024-11-11T16:37:52,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,733 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=142, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,733 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072727) roll requested 2024-11-11T16:37:52,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072733 2024-11-11T16:37:52,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072727 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072733 2024-11-11T16:37:52,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=143, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072733) roll requested 2024-11-11T16:37:52,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072740 2024-11-11T16:37:52,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072740 2024-11-11T16:37:52,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,747 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=144, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,747 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,747 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072740) roll requested 2024-11-11T16:37:52,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072747 2024-11-11T16:37:52,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072747 2024-11-11T16:37:52,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=145, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072747) roll requested 2024-11-11T16:37:52,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072756 2024-11-11T16:37:52,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072747 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072756 2024-11-11T16:37:52,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,762 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=146, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,763 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072756) roll requested 2024-11-11T16:37:52,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072763 2024-11-11T16:37:52,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072763 2024-11-11T16:37:52,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,769 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=147, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,769 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,769 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072763) roll requested 2024-11-11T16:37:52,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072770 2024-11-11T16:37:52,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072763 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072770 2024-11-11T16:37:52,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,778 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=148, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,778 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072770) roll requested 2024-11-11T16:37:52,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072778 2024-11-11T16:37:52,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072770 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072778 2024-11-11T16:37:52,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=149, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072778) roll requested 2024-11-11T16:37:52,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072785 2024-11-11T16:37:52,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072778 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072785 2024-11-11T16:37:52,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=150, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,792 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072785) roll requested 2024-11-11T16:37:52,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072792 2024-11-11T16:37:52,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072792 2024-11-11T16:37:52,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=151, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072792) roll requested 2024-11-11T16:37:52,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072799 2024-11-11T16:37:52,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072799 2024-11-11T16:37:52,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,805 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=152, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072799) roll requested 2024-11-11T16:37:52,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072805 2024-11-11T16:37:52,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072805 2024-11-11T16:37:52,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,812 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=153, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,812 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072805) roll requested 2024-11-11T16:37:52,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072812 2024-11-11T16:37:52,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072812 2024-11-11T16:37:52,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,818 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=154, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,818 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,818 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072812) roll requested 2024-11-11T16:37:52,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072818 2024-11-11T16:37:52,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072812 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072818 2024-11-11T16:37:52,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=155, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072818) roll requested 2024-11-11T16:37:52,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072824 2024-11-11T16:37:52,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072818 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072824 2024-11-11T16:37:52,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=156, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,831 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072824) roll requested 2024-11-11T16:37:52,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072832 2024-11-11T16:37:52,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072824 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072832 2024-11-11T16:37:52,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,838 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=157, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,838 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,838 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072832) roll requested 2024-11-11T16:37:52,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072838 2024-11-11T16:37:52,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072832 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072838 2024-11-11T16:37:52,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,844 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=158, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,844 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072838) roll requested 2024-11-11T16:37:52,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072845 2024-11-11T16:37:52,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072838 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072845 2024-11-11T16:37:52,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,851 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=159, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,851 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,851 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072845) roll requested 2024-11-11T16:37:52,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072851 2024-11-11T16:37:52,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072851 2024-11-11T16:37:52,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=160, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,858 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,858 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072851) roll requested 2024-11-11T16:37:52,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072858 2024-11-11T16:37:52,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072851 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072858 2024-11-11T16:37:52,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=161, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072858) roll requested 2024-11-11T16:37:52,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072865 2024-11-11T16:37:52,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072865 2024-11-11T16:37:52,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=162, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072865) roll requested 2024-11-11T16:37:52,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072871 2024-11-11T16:37:52,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072871 2024-11-11T16:37:52,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,877 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=163, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,877 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,877 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072871) roll requested 2024-11-11T16:37:52,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072877 2024-11-11T16:37:52,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072871 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072877 2024-11-11T16:37:52,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=164, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,884 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,884 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072877) roll requested 2024-11-11T16:37:52,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072884 2024-11-11T16:37:52,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072877 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072884 2024-11-11T16:37:52,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=165, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,891 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072884) roll requested 2024-11-11T16:37:52,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072891 2024-11-11T16:37:52,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072884 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072891 2024-11-11T16:37:52,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=166, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072891) roll requested 2024-11-11T16:37:52,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072897 2024-11-11T16:37:52,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072891 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072897 2024-11-11T16:37:52,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,905 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=167, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,905 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072897) roll requested 2024-11-11T16:37:52,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072905 2024-11-11T16:37:52,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072905 2024-11-11T16:37:52,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,911 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=168, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,911 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072905) roll requested 2024-11-11T16:37:52,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072911 2024-11-11T16:37:52,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072905 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072911 2024-11-11T16:37:52,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=169, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072911) roll requested 2024-11-11T16:37:52,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072921 2024-11-11T16:37:52,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072911 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072921 2024-11-11T16:37:52,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,927 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=170, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,928 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,928 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072921) roll requested 2024-11-11T16:37:52,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072928 2024-11-11T16:37:52,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072928 2024-11-11T16:37:52,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,934 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=171, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,934 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,934 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072928) roll requested 2024-11-11T16:37:52,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072934 2024-11-11T16:37:52,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072934 2024-11-11T16:37:52,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,941 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=172, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,941 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072934) roll requested 2024-11-11T16:37:52,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072941 2024-11-11T16:37:52,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072941 2024-11-11T16:37:52,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,948 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=173, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,948 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072941) roll requested 2024-11-11T16:37:52,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072948 2024-11-11T16:37:52,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072948 2024-11-11T16:37:52,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,955 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=174, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,955 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072948) roll requested 2024-11-11T16:37:52,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072955 2024-11-11T16:37:52,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072948 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072955 2024-11-11T16:37:52,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,962 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=175, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,962 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,962 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072955) roll requested 2024-11-11T16:37:52,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072962 2024-11-11T16:37:52,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072955 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072962 2024-11-11T16:37:52,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,969 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=176, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,969 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,969 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072962) roll requested 2024-11-11T16:37:52,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072969 2024-11-11T16:37:52,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072962 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072969 2024-11-11T16:37:52,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,976 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=177, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,976 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,976 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072969) roll requested 2024-11-11T16:37:52,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072976 2024-11-11T16:37:52,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072969 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072976 2024-11-11T16:37:52,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,983 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=178, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,983 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072976) roll requested 2024-11-11T16:37:52,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072983 2024-11-11T16:37:52,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072976 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072983 2024-11-11T16:37:52,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,990 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=179, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,990 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,990 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072983) roll requested 2024-11-11T16:37:52,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072990 2024-11-11T16:37:52,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:52,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072983 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072990 2024-11-11T16:37:52,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:52,996 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=180, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:52,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:52,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:52,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072990) roll requested 2024-11-11T16:37:52,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343072997 2024-11-11T16:37:53,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072990 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072997 2024-11-11T16:37:53,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=181, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,004 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343072997) roll requested 2024-11-11T16:37:53,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073004 2024-11-11T16:37:53,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343072997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073004 2024-11-11T16:37:53,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,010 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=182, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,010 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,010 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073004) roll requested 2024-11-11T16:37:53,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073010 2024-11-11T16:37:53,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073004 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073010 2024-11-11T16:37:53,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,016 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=183, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,016 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,016 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073010) roll requested 2024-11-11T16:37:53,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073016 2024-11-11T16:37:53,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073010 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073016 2024-11-11T16:37:53,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,023 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,023 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=184, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,023 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073016) roll requested 2024-11-11T16:37:53,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073023 2024-11-11T16:37:53,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073016 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073023 2024-11-11T16:37:53,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=185, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,029 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073023) roll requested 2024-11-11T16:37:53,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073030 2024-11-11T16:37:53,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073023 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073030 2024-11-11T16:37:53,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,036 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=186, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,036 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,036 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073030) roll requested 2024-11-11T16:37:53,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073036 2024-11-11T16:37:53,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073030 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073036 2024-11-11T16:37:53,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,043 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=187, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,043 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,043 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073036) roll requested 2024-11-11T16:37:53,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073043 2024-11-11T16:37:53,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073036 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073043 2024-11-11T16:37:53,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,059 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=188, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,059 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073043) roll requested 2024-11-11T16:37:53,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073059 2024-11-11T16:37:53,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073059 2024-11-11T16:37:53,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=189, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,068 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073059) roll requested 2024-11-11T16:37:53,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073068 2024-11-11T16:37:53,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073068 2024-11-11T16:37:53,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=190, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073068) roll requested 2024-11-11T16:37:53,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073077 2024-11-11T16:37:53,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073077 2024-11-11T16:37:53,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,089 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=191, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,089 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073077) roll requested 2024-11-11T16:37:53,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073090 2024-11-11T16:37:53,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073090 2024-11-11T16:37:53,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=192, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073090) roll requested 2024-11-11T16:37:53,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073107 2024-11-11T16:37:53,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073090 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073107 2024-11-11T16:37:53,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,117 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=193, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,117 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073107) roll requested 2024-11-11T16:37:53,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073117 2024-11-11T16:37:53,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073107 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073117 2024-11-11T16:37:53,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,125 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=194, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,125 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073117) roll requested 2024-11-11T16:37:53,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073125 2024-11-11T16:37:53,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073125 2024-11-11T16:37:53,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=195, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073125) roll requested 2024-11-11T16:37:53,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073133 2024-11-11T16:37:53,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073133 2024-11-11T16:37:53,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,141 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=196, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073133) roll requested 2024-11-11T16:37:53,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073141 2024-11-11T16:37:53,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073141 2024-11-11T16:37:53,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=197, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073141) roll requested 2024-11-11T16:37:53,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073149 2024-11-11T16:37:53,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073149 2024-11-11T16:37:53,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=198, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073149) roll requested 2024-11-11T16:37:53,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073155 2024-11-11T16:37:53,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073155 2024-11-11T16:37:53,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=199, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073155) roll requested 2024-11-11T16:37:53,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073165 2024-11-11T16:37:53,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073165 2024-11-11T16:37:53,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=200, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073165) roll requested 2024-11-11T16:37:53,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073173 2024-11-11T16:37:53,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073173 2024-11-11T16:37:53,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,181 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=201, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,181 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,181 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073173) roll requested 2024-11-11T16:37:53,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073181 2024-11-11T16:37:53,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073181 2024-11-11T16:37:53,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,191 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=202, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,191 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,191 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073181) roll requested 2024-11-11T16:37:53,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073191 2024-11-11T16:37:53,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073181 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073191 2024-11-11T16:37:53,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,199 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=203, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,199 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,199 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073191) roll requested 2024-11-11T16:37:53,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073200 2024-11-11T16:37:53,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073191 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073200 2024-11-11T16:37:53,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,206 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,206 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=204, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,206 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073200) roll requested 2024-11-11T16:37:53,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073206 2024-11-11T16:37:53,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073200 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073206 2024-11-11T16:37:53,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,213 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=205, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,213 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,213 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073206) roll requested 2024-11-11T16:37:53,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073213 2024-11-11T16:37:53,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073206 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073213 2024-11-11T16:37:53,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=206, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,219 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073213) roll requested 2024-11-11T16:37:53,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073219 2024-11-11T16:37:53,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073213 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073219 2024-11-11T16:37:53,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=207, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,226 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073219) roll requested 2024-11-11T16:37:53,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073226 2024-11-11T16:37:53,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073226 2024-11-11T16:37:53,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=208, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073226) roll requested 2024-11-11T16:37:53,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073233 2024-11-11T16:37:53,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073233 2024-11-11T16:37:53,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,241 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,241 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=209, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073233) roll requested 2024-11-11T16:37:53,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073241 2024-11-11T16:37:53,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073241 2024-11-11T16:37:53,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,249 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=210, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,249 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,249 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073241) roll requested 2024-11-11T16:37:53,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073249 2024-11-11T16:37:53,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073249 2024-11-11T16:37:53,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=211, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073249) roll requested 2024-11-11T16:37:53,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073255 2024-11-11T16:37:53,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073249 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073255 2024-11-11T16:37:53,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,262 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=212, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,262 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,262 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073255) roll requested 2024-11-11T16:37:53,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073262 2024-11-11T16:37:53,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073262 2024-11-11T16:37:53,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=213, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073262) roll requested 2024-11-11T16:37:53,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073268 2024-11-11T16:37:53,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073262 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073268 2024-11-11T16:37:53,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=214, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,276 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073268) roll requested 2024-11-11T16:37:53,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073276 2024-11-11T16:37:53,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073276 2024-11-11T16:37:53,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,283 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=215, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,283 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,283 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073276) roll requested 2024-11-11T16:37:53,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073283 2024-11-11T16:37:53,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073276 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073283 2024-11-11T16:37:53,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,290 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=216, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,290 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,290 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073283) roll requested 2024-11-11T16:37:53,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073290 2024-11-11T16:37:53,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073283 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073290 2024-11-11T16:37:53,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=217, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073290) roll requested 2024-11-11T16:37:53,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073297 2024-11-11T16:37:53,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073290 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073297 2024-11-11T16:37:53,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,303 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=218, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,303 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073297) roll requested 2024-11-11T16:37:53,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073303 2024-11-11T16:37:53,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073303 2024-11-11T16:37:53,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,310 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=219, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,310 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,310 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073303) roll requested 2024-11-11T16:37:53,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073310 2024-11-11T16:37:53,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073310 2024-11-11T16:37:53,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=220, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,317 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073310) roll requested 2024-11-11T16:37:53,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073317 2024-11-11T16:37:53,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073310 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073317 2024-11-11T16:37:53,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,323 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,323 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=221, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,323 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073317) roll requested 2024-11-11T16:37:53,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073324 2024-11-11T16:37:53,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073324 2024-11-11T16:37:53,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,330 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,330 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=222, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,330 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073324) roll requested 2024-11-11T16:37:53,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073330 2024-11-11T16:37:53,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073324 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073330 2024-11-11T16:37:53,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,337 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=223, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,337 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073330) roll requested 2024-11-11T16:37:53,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073337 2024-11-11T16:37:53,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073330 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073337 2024-11-11T16:37:53,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=224, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,344 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073337) roll requested 2024-11-11T16:37:53,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073344 2024-11-11T16:37:53,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073344 2024-11-11T16:37:53,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,351 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=225, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,351 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,352 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073344) roll requested 2024-11-11T16:37:53,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073352 2024-11-11T16:37:53,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073344 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073352 2024-11-11T16:37:53,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,358 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=226, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,358 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073352) roll requested 2024-11-11T16:37:53,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073358 2024-11-11T16:37:53,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073352 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073358 2024-11-11T16:37:53,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,366 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,366 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=227, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,366 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073358) roll requested 2024-11-11T16:37:53,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073367 2024-11-11T16:37:53,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073367 2024-11-11T16:37:53,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,373 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,373 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=228, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073367) roll requested 2024-11-11T16:37:53,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073373 2024-11-11T16:37:53,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073367 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073373 2024-11-11T16:37:53,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,379 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=229, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,379 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073373) roll requested 2024-11-11T16:37:53,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073380 2024-11-11T16:37:53,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073380 2024-11-11T16:37:53,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,386 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=230, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,386 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073380) roll requested 2024-11-11T16:37:53,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073387 2024-11-11T16:37:53,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073387 2024-11-11T16:37:53,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,393 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,394 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=231, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,394 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073387) roll requested 2024-11-11T16:37:53,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073394 2024-11-11T16:37:53,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073394 2024-11-11T16:37:53,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,401 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,401 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=232, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073394) roll requested 2024-11-11T16:37:53,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073401 2024-11-11T16:37:53,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073401 2024-11-11T16:37:53,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,409 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=233, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,409 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073401) roll requested 2024-11-11T16:37:53,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073409 2024-11-11T16:37:53,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073409 2024-11-11T16:37:53,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,416 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,416 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=234, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,416 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073409) roll requested 2024-11-11T16:37:53,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073416 2024-11-11T16:37:53,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073416 2024-11-11T16:37:53,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=235, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073416) roll requested 2024-11-11T16:37:53,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073424 2024-11-11T16:37:53,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073416 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073424 2024-11-11T16:37:53,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,434 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,434 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=236, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,434 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073424) roll requested 2024-11-11T16:37:53,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073434 2024-11-11T16:37:53,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073434 2024-11-11T16:37:53,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,442 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,442 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=237, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073434) roll requested 2024-11-11T16:37:53,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073442 2024-11-11T16:37:53,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073434 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073442 2024-11-11T16:37:53,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=238, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073442) roll requested 2024-11-11T16:37:53,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073450 2024-11-11T16:37:53,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073450 2024-11-11T16:37:53,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,461 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,461 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=239, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073450) roll requested 2024-11-11T16:37:53,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073461 2024-11-11T16:37:53,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073461 2024-11-11T16:37:53,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,469 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=240, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,470 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073461) roll requested 2024-11-11T16:37:53,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073470 2024-11-11T16:37:53,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073470 2024-11-11T16:37:53,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=241, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,478 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073470) roll requested 2024-11-11T16:37:53,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073478 2024-11-11T16:37:53,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073470 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073478 2024-11-11T16:37:53,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,488 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=242, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,488 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,489 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073478) roll requested 2024-11-11T16:37:53,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073489 2024-11-11T16:37:53,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073489 2024-11-11T16:37:53,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,506 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,506 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=243, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,507 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073489) roll requested 2024-11-11T16:37:53,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073507 2024-11-11T16:37:53,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073489 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073507 2024-11-11T16:37:53,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,514 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=244, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,514 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,515 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073507) roll requested 2024-11-11T16:37:53,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073515 2024-11-11T16:37:53,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073507 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073515 2024-11-11T16:37:53,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=245, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073515) roll requested 2024-11-11T16:37:53,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073523 2024-11-11T16:37:53,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073515 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073523 2024-11-11T16:37:53,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,530 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=246, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,531 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073523) roll requested 2024-11-11T16:37:53,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073531 2024-11-11T16:37:53,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073531 2024-11-11T16:37:53,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,538 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=247, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,538 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,538 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073531) roll requested 2024-11-11T16:37:53,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073539 2024-11-11T16:37:53,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073539 2024-11-11T16:37:53,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,546 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=248, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,546 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,546 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073539) roll requested 2024-11-11T16:37:53,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073547 2024-11-11T16:37:53,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073539 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073547 2024-11-11T16:37:53,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,556 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=249, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073547) roll requested 2024-11-11T16:37:53,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073557 2024-11-11T16:37:53,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073547 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073557 2024-11-11T16:37:53,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=250, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,565 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073557) roll requested 2024-11-11T16:37:53,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073565 2024-11-11T16:37:53,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073565 2024-11-11T16:37:53,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=251, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,582 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073565) roll requested 2024-11-11T16:37:53,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073583 2024-11-11T16:37:53,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073583 2024-11-11T16:37:53,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,601 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=252, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073583) roll requested 2024-11-11T16:37:53,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073609 2024-11-11T16:37:53,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073583 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073609 2024-11-11T16:37:53,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,627 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=253, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,627 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073609) roll requested 2024-11-11T16:37:53,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073629 2024-11-11T16:37:53,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073629 2024-11-11T16:37:53,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=254, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,643 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,643 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073629) roll requested 2024-11-11T16:37:53,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073644 2024-11-11T16:37:53,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073644 2024-11-11T16:37:53,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,655 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=255, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,656 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073644) roll requested 2024-11-11T16:37:53,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073656 2024-11-11T16:37:53,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073644 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073656 2024-11-11T16:37:53,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=256, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,665 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073656) roll requested 2024-11-11T16:37:53,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073665 2024-11-11T16:37:53,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073656 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073665 2024-11-11T16:37:53,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,672 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=257, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,672 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073665) roll requested 2024-11-11T16:37:53,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073672 2024-11-11T16:37:53,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073672 2024-11-11T16:37:53,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,679 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=258, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,679 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,679 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073672) roll requested 2024-11-11T16:37:53,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073679 2024-11-11T16:37:53,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073672 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073679 2024-11-11T16:37:53,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=259, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073679) roll requested 2024-11-11T16:37:53,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073687 2024-11-11T16:37:53,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073679 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073687 2024-11-11T16:37:53,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=260, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073687) roll requested 2024-11-11T16:37:53,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073694 2024-11-11T16:37:53,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073694 2024-11-11T16:37:53,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=261, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073694) roll requested 2024-11-11T16:37:53,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073701 2024-11-11T16:37:53,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073701 2024-11-11T16:37:53,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=262, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073701) roll requested 2024-11-11T16:37:53,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073708 2024-11-11T16:37:53,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073708 2024-11-11T16:37:53,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=263, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073708) roll requested 2024-11-11T16:37:53,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073716 2024-11-11T16:37:53,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073716 2024-11-11T16:37:53,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=264, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073716) roll requested 2024-11-11T16:37:53,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073724 2024-11-11T16:37:53,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073724 2024-11-11T16:37:53,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,735 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=265, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,735 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,735 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073724) roll requested 2024-11-11T16:37:53,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073735 2024-11-11T16:37:53,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073735 2024-11-11T16:37:53,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,745 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,745 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=266, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,745 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073735) roll requested 2024-11-11T16:37:53,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073745 2024-11-11T16:37:53,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073735 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073745 2024-11-11T16:37:53,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,755 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,755 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=267, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,755 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073745) roll requested 2024-11-11T16:37:53,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073755 2024-11-11T16:37:53,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073755 2024-11-11T16:37:53,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=268, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,762 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073755) roll requested 2024-11-11T16:37:53,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073762 2024-11-11T16:37:53,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073755 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073762 2024-11-11T16:37:53,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=269, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073762) roll requested 2024-11-11T16:37:53,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073768 2024-11-11T16:37:53,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073762 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073768 2024-11-11T16:37:53,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,774 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,774 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=270, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,774 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073768) roll requested 2024-11-11T16:37:53,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073774 2024-11-11T16:37:53,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073774 2024-11-11T16:37:53,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,781 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=271, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073774) roll requested 2024-11-11T16:37:53,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073782 2024-11-11T16:37:53,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073774 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073782 2024-11-11T16:37:53,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=272, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073782) roll requested 2024-11-11T16:37:53,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073788 2024-11-11T16:37:53,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073788 2024-11-11T16:37:53,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,794 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,794 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=273, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073788) roll requested 2024-11-11T16:37:53,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073795 2024-11-11T16:37:53,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073795 2024-11-11T16:37:53,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=274, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073795) roll requested 2024-11-11T16:37:53,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073806 2024-11-11T16:37:53,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073795 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073806 2024-11-11T16:37:53,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,814 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=275, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,814 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073806) roll requested 2024-11-11T16:37:53,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073814 2024-11-11T16:37:53,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073814 2024-11-11T16:37:53,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=276, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073814) roll requested 2024-11-11T16:37:53,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073821 2024-11-11T16:37:53,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073814 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073821 2024-11-11T16:37:53,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=277, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073821) roll requested 2024-11-11T16:37:53,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073831 2024-11-11T16:37:53,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073831 2024-11-11T16:37:53,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,841 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,841 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=278, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073831) roll requested 2024-11-11T16:37:53,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073842 2024-11-11T16:37:53,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073842 2024-11-11T16:37:53,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,853 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=279, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,853 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073842) roll requested 2024-11-11T16:37:53,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073854 2024-11-11T16:37:53,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073854 2024-11-11T16:37:53,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=280, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073854) roll requested 2024-11-11T16:37:53,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073860 2024-11-11T16:37:53,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073860 2024-11-11T16:37:53,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,866 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=281, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,866 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073860) roll requested 2024-11-11T16:37:53,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073867 2024-11-11T16:37:53,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073867 2024-11-11T16:37:53,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=282, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073867) roll requested 2024-11-11T16:37:53,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073873 2024-11-11T16:37:53,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073867 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073873 2024-11-11T16:37:53,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=283, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073873) roll requested 2024-11-11T16:37:53,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073881 2024-11-11T16:37:53,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073881 2024-11-11T16:37:53,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=284, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,887 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073881) roll requested 2024-11-11T16:37:53,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073888 2024-11-11T16:37:53,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073888 2024-11-11T16:37:53,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,895 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,895 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=285, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,895 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073888) roll requested 2024-11-11T16:37:53,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073895 2024-11-11T16:37:53,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073895 2024-11-11T16:37:53,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,902 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,902 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=286, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073895) roll requested 2024-11-11T16:37:53,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073902 2024-11-11T16:37:53,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073895 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073902 2024-11-11T16:37:53,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,910 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,910 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=287, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,910 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073902) roll requested 2024-11-11T16:37:53,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073910 2024-11-11T16:37:53,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073910 2024-11-11T16:37:53,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=288, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,918 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073910) roll requested 2024-11-11T16:37:53,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073918 2024-11-11T16:37:53,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073910 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073918 2024-11-11T16:37:53,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,926 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,926 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=289, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,926 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073918) roll requested 2024-11-11T16:37:53,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073926 2024-11-11T16:37:53,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073918 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073926 2024-11-11T16:37:53,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=290, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073926) roll requested 2024-11-11T16:37:53,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073936 2024-11-11T16:37:53,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073926 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073936 2024-11-11T16:37:53,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,944 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=291, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073936) roll requested 2024-11-11T16:37:53,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073945 2024-11-11T16:37:53,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073945 2024-11-11T16:37:53,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,953 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,953 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=292, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073945) roll requested 2024-11-11T16:37:53,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073953 2024-11-11T16:37:53,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073945 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073953 2024-11-11T16:37:53,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,962 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,962 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=293, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,962 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073953) roll requested 2024-11-11T16:37:53,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073962 2024-11-11T16:37:53,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073962 2024-11-11T16:37:53,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,970 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,970 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=294, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,970 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073962) roll requested 2024-11-11T16:37:53,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073970 2024-11-11T16:37:53,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073962 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073970 2024-11-11T16:37:53,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,978 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=295, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,978 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073970) roll requested 2024-11-11T16:37:53,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073978 2024-11-11T16:37:53,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073970 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073978 2024-11-11T16:37:53,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=296, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,986 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073978) roll requested 2024-11-11T16:37:53,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073986 2024-11-11T16:37:53,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:53,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073986 2024-11-11T16:37:53,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:53,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=297, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:53,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:53,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:53,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073986) roll requested 2024-11-11T16:37:53,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343073993 2024-11-11T16:37:54,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073993 2024-11-11T16:37:54,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=298, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,001 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343073993) roll requested 2024-11-11T16:37:54,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074001 2024-11-11T16:37:54,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343073993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074001 2024-11-11T16:37:54,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=299, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,009 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074001) roll requested 2024-11-11T16:37:54,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074009 2024-11-11T16:37:54,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074001 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074009 2024-11-11T16:37:54,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=300, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074009) roll requested 2024-11-11T16:37:54,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074025 2024-11-11T16:37:54,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074009 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074025 2024-11-11T16:37:54,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,036 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=301, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074025) roll requested 2024-11-11T16:37:54,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074037 2024-11-11T16:37:54,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074037 2024-11-11T16:37:54,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,046 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=302, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,046 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,046 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074037) roll requested 2024-11-11T16:37:54,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074046 2024-11-11T16:37:54,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074046 2024-11-11T16:37:54,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,053 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=303, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,053 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074046) roll requested 2024-11-11T16:37:54,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074053 2024-11-11T16:37:54,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074046 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074053 2024-11-11T16:37:54,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=304, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,060 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074053) roll requested 2024-11-11T16:37:54,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074060 2024-11-11T16:37:54,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074053 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074060 2024-11-11T16:37:54,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,073 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=305, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074060) roll requested 2024-11-11T16:37:54,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074073 2024-11-11T16:37:54,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074073 2024-11-11T16:37:54,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,085 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=306, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,085 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074073) roll requested 2024-11-11T16:37:54,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074085 2024-11-11T16:37:54,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074085 2024-11-11T16:37:54,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,093 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,093 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=307, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,093 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074085) roll requested 2024-11-11T16:37:54,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074093 2024-11-11T16:37:54,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074093 2024-11-11T16:37:54,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,100 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,100 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=308, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,101 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074093) roll requested 2024-11-11T16:37:54,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074101 2024-11-11T16:37:54,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074101 2024-11-11T16:37:54,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,108 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=309, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,108 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074101) roll requested 2024-11-11T16:37:54,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074108 2024-11-11T16:37:54,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074108 2024-11-11T16:37:54,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,117 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,117 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=310, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074108) roll requested 2024-11-11T16:37:54,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074117 2024-11-11T16:37:54,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074108 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074117 2024-11-11T16:37:54,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,124 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=311, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,124 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074117) roll requested 2024-11-11T16:37:54,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074124 2024-11-11T16:37:54,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074124 2024-11-11T16:37:54,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,131 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,131 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=312, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,131 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074124) roll requested 2024-11-11T16:37:54,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074131 2024-11-11T16:37:54,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074124 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074131 2024-11-11T16:37:54,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,138 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,139 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=313, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,139 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074131) roll requested 2024-11-11T16:37:54,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074139 2024-11-11T16:37:54,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074131 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074139 2024-11-11T16:37:54,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,146 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,146 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=314, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,147 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074139) roll requested 2024-11-11T16:37:54,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074147 2024-11-11T16:37:54,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074147 2024-11-11T16:37:54,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,154 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,154 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=315, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,154 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074147) roll requested 2024-11-11T16:37:54,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074154 2024-11-11T16:37:54,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074147 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074154 2024-11-11T16:37:54,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,160 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=316, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074154) roll requested 2024-11-11T16:37:54,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074161 2024-11-11T16:37:54,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074154 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074161 2024-11-11T16:37:54,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=317, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074161) roll requested 2024-11-11T16:37:54,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074168 2024-11-11T16:37:54,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074168 2024-11-11T16:37:54,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,176 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,176 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=318, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,176 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074168) roll requested 2024-11-11T16:37:54,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074176 2024-11-11T16:37:54,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074176 2024-11-11T16:37:54,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,183 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=319, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,183 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,183 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074176) roll requested 2024-11-11T16:37:54,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074183 2024-11-11T16:37:54,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074176 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074183 2024-11-11T16:37:54,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,191 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,191 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=320, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,191 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074183) roll requested 2024-11-11T16:37:54,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074191 2024-11-11T16:37:54,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074183 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074191 2024-11-11T16:37:54,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,198 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=321, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074191) roll requested 2024-11-11T16:37:54,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074198 2024-11-11T16:37:54,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074191 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074198 2024-11-11T16:37:54,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,205 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,206 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=322, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,206 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074198) roll requested 2024-11-11T16:37:54,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074206 2024-11-11T16:37:54,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074206 2024-11-11T16:37:54,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,213 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,213 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=323, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,213 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074206) roll requested 2024-11-11T16:37:54,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074213 2024-11-11T16:37:54,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074206 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074213 2024-11-11T16:37:54,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,219 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=324, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074213) roll requested 2024-11-11T16:37:54,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074219 2024-11-11T16:37:54,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074213 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074219 2024-11-11T16:37:54,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,226 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=325, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074219) roll requested 2024-11-11T16:37:54,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074226 2024-11-11T16:37:54,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074226 2024-11-11T16:37:54,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=326, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074226) roll requested 2024-11-11T16:37:54,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074232 2024-11-11T16:37:54,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074232 2024-11-11T16:37:54,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,241 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=327, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074232) roll requested 2024-11-11T16:37:54,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074241 2024-11-11T16:37:54,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074232 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074241 2024-11-11T16:37:54,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=328, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074241) roll requested 2024-11-11T16:37:54,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074247 2024-11-11T16:37:54,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074247 2024-11-11T16:37:54,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=329, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074247) roll requested 2024-11-11T16:37:54,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074255 2024-11-11T16:37:54,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074255 2024-11-11T16:37:54,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=330, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074255) roll requested 2024-11-11T16:37:54,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074268 2024-11-11T16:37:54,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074268 2024-11-11T16:37:54,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=331, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,276 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074268) roll requested 2024-11-11T16:37:54,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074276 2024-11-11T16:37:54,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074276 2024-11-11T16:37:54,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,283 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,283 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=332, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,283 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074276) roll requested 2024-11-11T16:37:54,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074285 2024-11-11T16:37:54,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074276 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074285 2024-11-11T16:37:54,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,291 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,291 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=333, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074285) roll requested 2024-11-11T16:37:54,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074292 2024-11-11T16:37:54,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074292 2024-11-11T16:37:54,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,298 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,298 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=334, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,298 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074292) roll requested 2024-11-11T16:37:54,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074299 2024-11-11T16:37:54,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074299 2024-11-11T16:37:54,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,305 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=335, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074299) roll requested 2024-11-11T16:37:54,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074305 2024-11-11T16:37:54,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074305 2024-11-11T16:37:54,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,313 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=336, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,313 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074305) roll requested 2024-11-11T16:37:54,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074313 2024-11-11T16:37:54,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074313 2024-11-11T16:37:54,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=337, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074313) roll requested 2024-11-11T16:37:54,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074320 2024-11-11T16:37:54,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074320 2024-11-11T16:37:54,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=338, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,327 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074320) roll requested 2024-11-11T16:37:54,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074327 2024-11-11T16:37:54,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074327 2024-11-11T16:37:54,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=339, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,334 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074327) roll requested 2024-11-11T16:37:54,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074334 2024-11-11T16:37:54,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074327 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074334 2024-11-11T16:37:54,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,343 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,343 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=340, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,343 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074334) roll requested 2024-11-11T16:37:54,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074343 2024-11-11T16:37:54,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074343 2024-11-11T16:37:54,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,352 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=341, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,352 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,352 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074343) roll requested 2024-11-11T16:37:54,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074352 2024-11-11T16:37:54,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074343 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074352 2024-11-11T16:37:54,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,359 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,359 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=342, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,359 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074352) roll requested 2024-11-11T16:37:54,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074359 2024-11-11T16:37:54,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074352 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074359 2024-11-11T16:37:54,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,366 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,366 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=343, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,366 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074359) roll requested 2024-11-11T16:37:54,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074366 2024-11-11T16:37:54,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074359 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074366 2024-11-11T16:37:54,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,373 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,373 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=344, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074366) roll requested 2024-11-11T16:37:54,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074373 2024-11-11T16:37:54,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074366 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074373 2024-11-11T16:37:54,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,379 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,379 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=345, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074373) roll requested 2024-11-11T16:37:54,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074380 2024-11-11T16:37:54,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074380 2024-11-11T16:37:54,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,386 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=346, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,386 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074380) roll requested 2024-11-11T16:37:54,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074386 2024-11-11T16:37:54,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074386 2024-11-11T16:37:54,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,393 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,393 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=347, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,393 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074386) roll requested 2024-11-11T16:37:54,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074393 2024-11-11T16:37:54,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074386 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074393 2024-11-11T16:37:54,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=348, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074393) roll requested 2024-11-11T16:37:54,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074400 2024-11-11T16:37:54,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074393 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074400 2024-11-11T16:37:54,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,407 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,407 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=349, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,407 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074400) roll requested 2024-11-11T16:37:54,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074407 2024-11-11T16:37:54,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074407 2024-11-11T16:37:54,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,413 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,413 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=350, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,413 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074407) roll requested 2024-11-11T16:37:54,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074413 2024-11-11T16:37:54,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074413 2024-11-11T16:37:54,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=351, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074413) roll requested 2024-11-11T16:37:54,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074420 2024-11-11T16:37:54,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074413 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074420 2024-11-11T16:37:54,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,427 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=352, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,427 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074420) roll requested 2024-11-11T16:37:54,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074427 2024-11-11T16:37:54,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074427 2024-11-11T16:37:54,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,434 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,434 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=353, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,434 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074427) roll requested 2024-11-11T16:37:54,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074434 2024-11-11T16:37:54,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074427 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074434 2024-11-11T16:37:54,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,441 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=354, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,441 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074434) roll requested 2024-11-11T16:37:54,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074441 2024-11-11T16:37:54,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074434 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074441 2024-11-11T16:37:54,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,447 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,447 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=355, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,447 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074441) roll requested 2024-11-11T16:37:54,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074447 2024-11-11T16:37:54,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074441 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074447 2024-11-11T16:37:54,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,454 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,454 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=356, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,454 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074447) roll requested 2024-11-11T16:37:54,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074455 2024-11-11T16:37:54,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074447 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074455 2024-11-11T16:37:54,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,462 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,462 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=357, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,462 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074455) roll requested 2024-11-11T16:37:54,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074462 2024-11-11T16:37:54,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074455 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074462 2024-11-11T16:37:54,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,469 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=358, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074462) roll requested 2024-11-11T16:37:54,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074469 2024-11-11T16:37:54,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074469 2024-11-11T16:37:54,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,476 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=359, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074469) roll requested 2024-11-11T16:37:54,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074477 2024-11-11T16:37:54,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074477 2024-11-11T16:37:54,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=360, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074477) roll requested 2024-11-11T16:37:54,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074485 2024-11-11T16:37:54,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074485 2024-11-11T16:37:54,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=361, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074485) roll requested 2024-11-11T16:37:54,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074492 2024-11-11T16:37:54,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074492 2024-11-11T16:37:54,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,501 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=362, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074492) roll requested 2024-11-11T16:37:54,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074501 2024-11-11T16:37:54,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074501 2024-11-11T16:37:54,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,507 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=363, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074501) roll requested 2024-11-11T16:37:54,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074508 2024-11-11T16:37:54,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074508 2024-11-11T16:37:54,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,515 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,515 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=364, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,515 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074508) roll requested 2024-11-11T16:37:54,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074515 2024-11-11T16:37:54,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074508 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074515 2024-11-11T16:37:54,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,522 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=365, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,522 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074515) roll requested 2024-11-11T16:37:54,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074522 2024-11-11T16:37:54,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074515 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074522 2024-11-11T16:37:54,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=366, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074522) roll requested 2024-11-11T16:37:54,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074529 2024-11-11T16:37:54,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074529 2024-11-11T16:37:54,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,535 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=367, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,536 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074529) roll requested 2024-11-11T16:37:54,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074536 2024-11-11T16:37:54,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074536 2024-11-11T16:37:54,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=368, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074536) roll requested 2024-11-11T16:37:54,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074543 2024-11-11T16:37:54,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074543 2024-11-11T16:37:54,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=369, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074543) roll requested 2024-11-11T16:37:54,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074550 2024-11-11T16:37:54,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074550 2024-11-11T16:37:54,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,556 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=370, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,556 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074550) roll requested 2024-11-11T16:37:54,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074556 2024-11-11T16:37:54,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074556 2024-11-11T16:37:54,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,565 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=371, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,565 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074556) roll requested 2024-11-11T16:37:54,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074565 2024-11-11T16:37:54,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074556 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074565 2024-11-11T16:37:54,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=372, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074565) roll requested 2024-11-11T16:37:54,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074572 2024-11-11T16:37:54,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074572 2024-11-11T16:37:54,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,579 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=373, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,579 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074572) roll requested 2024-11-11T16:37:54,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074579 2024-11-11T16:37:54,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074579 2024-11-11T16:37:54,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,587 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=374, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,587 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074579) roll requested 2024-11-11T16:37:54,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074587 2024-11-11T16:37:54,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074579 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074587 2024-11-11T16:37:54,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,594 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,594 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=375, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,594 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074587) roll requested 2024-11-11T16:37:54,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074594 2024-11-11T16:37:54,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074594 2024-11-11T16:37:54,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,601 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,601 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=376, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074594) roll requested 2024-11-11T16:37:54,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074601 2024-11-11T16:37:54,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074594 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074601 2024-11-11T16:37:54,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=377, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074601) roll requested 2024-11-11T16:37:54,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074608 2024-11-11T16:37:54,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074608 2024-11-11T16:37:54,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,614 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,614 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=378, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,614 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074608) roll requested 2024-11-11T16:37:54,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074614 2024-11-11T16:37:54,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074614 2024-11-11T16:37:54,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,620 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,620 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=379, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,620 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074614) roll requested 2024-11-11T16:37:54,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074620 2024-11-11T16:37:54,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074614 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074620 2024-11-11T16:37:54,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,626 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,626 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=380, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,627 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074620) roll requested 2024-11-11T16:37:54,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074627 2024-11-11T16:37:54,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074627 2024-11-11T16:37:54,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=381, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074627) roll requested 2024-11-11T16:37:54,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074633 2024-11-11T16:37:54,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074627 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074633 2024-11-11T16:37:54,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,639 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,639 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=382, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,639 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074633) roll requested 2024-11-11T16:37:54,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074639 2024-11-11T16:37:54,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074639 2024-11-11T16:37:54,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,646 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=383, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074639) roll requested 2024-11-11T16:37:54,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074646 2024-11-11T16:37:54,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074639 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074646 2024-11-11T16:37:54,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,653 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=384, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,653 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,653 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074646) roll requested 2024-11-11T16:37:54,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074654 2024-11-11T16:37:54,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074646 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074654 2024-11-11T16:37:54,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,661 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=385, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,661 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,661 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074654) roll requested 2024-11-11T16:37:54,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074661 2024-11-11T16:37:54,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074661 2024-11-11T16:37:54,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,667 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,667 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=386, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074661) roll requested 2024-11-11T16:37:54,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074667 2024-11-11T16:37:54,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074661 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074667 2024-11-11T16:37:54,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=387, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074667) roll requested 2024-11-11T16:37:54,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074674 2024-11-11T16:37:54,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074674 2024-11-11T16:37:54,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,680 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=388, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,680 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074674) roll requested 2024-11-11T16:37:54,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074680 2024-11-11T16:37:54,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074680 2024-11-11T16:37:54,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=389, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074680) roll requested 2024-11-11T16:37:54,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074687 2024-11-11T16:37:54,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074680 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074687 2024-11-11T16:37:54,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,697 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=390, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,697 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074687) roll requested 2024-11-11T16:37:54,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074697 2024-11-11T16:37:54,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074697 2024-11-11T16:37:54,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,705 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,705 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=391, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,705 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074697) roll requested 2024-11-11T16:37:54,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074705 2024-11-11T16:37:54,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074697 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074705 2024-11-11T16:37:54,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,712 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,712 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=392, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,713 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074705) roll requested 2024-11-11T16:37:54,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074713 2024-11-11T16:37:54,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074705 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074713 2024-11-11T16:37:54,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,719 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,719 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=393, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,719 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074713) roll requested 2024-11-11T16:37:54,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074719 2024-11-11T16:37:54,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074713 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074719 2024-11-11T16:37:54,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,725 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=394, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074719) roll requested 2024-11-11T16:37:54,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074726 2024-11-11T16:37:54,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074719 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074726 2024-11-11T16:37:54,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,733 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=395, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,733 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074726) roll requested 2024-11-11T16:37:54,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074733 2024-11-11T16:37:54,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074733 2024-11-11T16:37:54,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,739 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=396, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,739 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074733) roll requested 2024-11-11T16:37:54,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074739 2024-11-11T16:37:54,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074739 2024-11-11T16:37:54,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=397, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074739) roll requested 2024-11-11T16:37:54,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074761 2024-11-11T16:37:54,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074739 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074761 2024-11-11T16:37:54,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=398, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074761) roll requested 2024-11-11T16:37:54,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074773 2024-11-11T16:37:54,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074773 2024-11-11T16:37:54,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,780 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,780 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=399, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,780 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074773) roll requested 2024-11-11T16:37:54,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074780 2024-11-11T16:37:54,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074780 2024-11-11T16:37:54,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,787 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,787 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=400, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,787 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074780) roll requested 2024-11-11T16:37:54,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074787 2024-11-11T16:37:54,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074780 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074787 2024-11-11T16:37:54,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,794 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=401, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074787) roll requested 2024-11-11T16:37:54,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074794 2024-11-11T16:37:54,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074787 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074794 2024-11-11T16:37:54,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,801 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=402, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074794) roll requested 2024-11-11T16:37:54,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074802 2024-11-11T16:37:54,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074802 2024-11-11T16:37:54,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=403, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074802) roll requested 2024-11-11T16:37:54,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074808 2024-11-11T16:37:54,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074808 2024-11-11T16:37:54,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=404, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074808) roll requested 2024-11-11T16:37:54,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074815 2024-11-11T16:37:54,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074815 2024-11-11T16:37:54,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=405, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074815) roll requested 2024-11-11T16:37:54,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074821 2024-11-11T16:37:54,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074821 2024-11-11T16:37:54,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=406, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074821) roll requested 2024-11-11T16:37:54,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074828 2024-11-11T16:37:54,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074828 2024-11-11T16:37:54,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,834 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,834 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=407, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,834 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074828) roll requested 2024-11-11T16:37:54,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074834 2024-11-11T16:37:54,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074834 2024-11-11T16:37:54,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,840 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=408, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,841 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074834) roll requested 2024-11-11T16:37:54,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074841 2024-11-11T16:37:54,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074834 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074841 2024-11-11T16:37:54,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=409, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074841) roll requested 2024-11-11T16:37:54,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074847 2024-11-11T16:37:54,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074841 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074847 2024-11-11T16:37:54,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=410, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074847) roll requested 2024-11-11T16:37:54,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074854 2024-11-11T16:37:54,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074854 2024-11-11T16:37:54,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=411, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074854) roll requested 2024-11-11T16:37:54,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074861 2024-11-11T16:37:54,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074861 2024-11-11T16:37:54,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,867 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,867 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=412, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,867 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074861) roll requested 2024-11-11T16:37:54,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074867 2024-11-11T16:37:54,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074867 2024-11-11T16:37:54,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=413, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074867) roll requested 2024-11-11T16:37:54,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074873 2024-11-11T16:37:54,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074867 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074873 2024-11-11T16:37:54,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=414, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074873) roll requested 2024-11-11T16:37:54,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074880 2024-11-11T16:37:54,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074880 2024-11-11T16:37:54,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,886 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,886 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=415, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074880) roll requested 2024-11-11T16:37:54,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074887 2024-11-11T16:37:54,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074887 2024-11-11T16:37:54,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=416, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074887) roll requested 2024-11-11T16:37:54,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074893 2024-11-11T16:37:54,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074893 2024-11-11T16:37:54,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,899 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=417, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,899 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074893) roll requested 2024-11-11T16:37:54,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074899 2024-11-11T16:37:54,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074899 2024-11-11T16:37:54,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=418, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074899) roll requested 2024-11-11T16:37:54,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074907 2024-11-11T16:37:54,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074899 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074907 2024-11-11T16:37:54,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=419, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074907) roll requested 2024-11-11T16:37:54,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074915 2024-11-11T16:37:54,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074915 2024-11-11T16:37:54,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,922 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,922 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=420, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,922 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074915) roll requested 2024-11-11T16:37:54,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074923 2024-11-11T16:37:54,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074923 2024-11-11T16:37:54,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,929 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=421, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,929 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074923) roll requested 2024-11-11T16:37:54,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074929 2024-11-11T16:37:54,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074923 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074929 2024-11-11T16:37:54,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,935 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=422, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074929) roll requested 2024-11-11T16:37:54,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074936 2024-11-11T16:37:54,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074929 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074936 2024-11-11T16:37:54,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,942 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,942 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=423, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,942 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074936) roll requested 2024-11-11T16:37:54,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074942 2024-11-11T16:37:54,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074942 2024-11-11T16:37:54,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,949 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=424, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074942) roll requested 2024-11-11T16:37:54,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074949 2024-11-11T16:37:54,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074942 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074949 2024-11-11T16:37:54,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=425, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074949) roll requested 2024-11-11T16:37:54,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074965 2024-11-11T16:37:54,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074965 2024-11-11T16:37:54,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,973 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=426, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074965) roll requested 2024-11-11T16:37:54,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074973 2024-11-11T16:37:54,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074973 2024-11-11T16:37:54,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,980 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,980 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=427, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,980 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074973) roll requested 2024-11-11T16:37:54,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074980 2024-11-11T16:37:54,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074980 2024-11-11T16:37:54,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,987 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,987 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=428, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,987 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074980) roll requested 2024-11-11T16:37:54,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074987 2024-11-11T16:37:54,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:54,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074980 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074987 2024-11-11T16:37:54,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:54,995 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,995 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=429, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:54,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:54,995 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:54,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074987) roll requested 2024-11-11T16:37:54,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343074995 2024-11-11T16:37:55,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074995 2024-11-11T16:37:55,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=430, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,001 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343074995) roll requested 2024-11-11T16:37:55,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075001 2024-11-11T16:37:55,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343074995 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075001 2024-11-11T16:37:55,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=431, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075001) roll requested 2024-11-11T16:37:55,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075012 2024-11-11T16:37:55,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075001 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075012 2024-11-11T16:37:55,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=432, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075012) roll requested 2024-11-11T16:37:55,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075018 2024-11-11T16:37:55,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075018 2024-11-11T16:37:55,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=433, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075018) roll requested 2024-11-11T16:37:55,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075025 2024-11-11T16:37:55,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075025 2024-11-11T16:37:55,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,035 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,035 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=434, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,036 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075025) roll requested 2024-11-11T16:37:55,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075036 2024-11-11T16:37:55,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075036 2024-11-11T16:37:55,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,046 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,046 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=435, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,046 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075036) roll requested 2024-11-11T16:37:55,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075046 2024-11-11T16:37:55,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075036 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075046 2024-11-11T16:37:55,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=436, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075046) roll requested 2024-11-11T16:37:55,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075055 2024-11-11T16:37:55,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075046 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075055 2024-11-11T16:37:55,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,062 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,063 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=437, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,063 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075055) roll requested 2024-11-11T16:37:55,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075063 2024-11-11T16:37:55,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075063 2024-11-11T16:37:55,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,071 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,071 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=438, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,071 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075063) roll requested 2024-11-11T16:37:55,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075071 2024-11-11T16:37:55,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075071 2024-11-11T16:37:55,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=439, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,081 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075071) roll requested 2024-11-11T16:37:55,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075081 2024-11-11T16:37:55,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075071 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075081 2024-11-11T16:37:55,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,091 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,091 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=440, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,091 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075081) roll requested 2024-11-11T16:37:55,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075091 2024-11-11T16:37:55,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075081 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075091 2024-11-11T16:37:55,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=441, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075091) roll requested 2024-11-11T16:37:55,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075105 2024-11-11T16:37:55,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075105 2024-11-11T16:37:55,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,121 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=442, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075105) roll requested 2024-11-11T16:37:55,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075121 2024-11-11T16:37:55,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075121 2024-11-11T16:37:55,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=443, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,137 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075121) roll requested 2024-11-11T16:37:55,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075137 2024-11-11T16:37:55,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075137 2024-11-11T16:37:55,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,151 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,151 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=444, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,151 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075137) roll requested 2024-11-11T16:37:55,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075151 2024-11-11T16:37:55,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075137 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075151 2024-11-11T16:37:55,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=445, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075151) roll requested 2024-11-11T16:37:55,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075165 2024-11-11T16:37:55,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075151 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075165 2024-11-11T16:37:55,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=446, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075165) roll requested 2024-11-11T16:37:55,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075174 2024-11-11T16:37:55,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075174 2024-11-11T16:37:55,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=447, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075174) roll requested 2024-11-11T16:37:55,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075184 2024-11-11T16:37:55,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075184 2024-11-11T16:37:55,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,192 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=448, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,192 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,192 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075184) roll requested 2024-11-11T16:37:55,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075192 2024-11-11T16:37:55,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075192 2024-11-11T16:37:55,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,201 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=449, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,201 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,201 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075192) roll requested 2024-11-11T16:37:55,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075201 2024-11-11T16:37:55,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075192 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075201 2024-11-11T16:37:55,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,213 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,213 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=450, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,213 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075201) roll requested 2024-11-11T16:37:55,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075213 2024-11-11T16:37:55,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075213 2024-11-11T16:37:55,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,228 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=451, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075213) roll requested 2024-11-11T16:37:55,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075229 2024-11-11T16:37:55,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075213 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075229 2024-11-11T16:37:55,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,236 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=452, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,236 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075229) roll requested 2024-11-11T16:37:55,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075237 2024-11-11T16:37:55,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075237 2024-11-11T16:37:55,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=453, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,244 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075237) roll requested 2024-11-11T16:37:55,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075244 2024-11-11T16:37:55,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075244 2024-11-11T16:37:55,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,253 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,253 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=454, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075244) roll requested 2024-11-11T16:37:55,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075253 2024-11-11T16:37:55,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075244 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075253 2024-11-11T16:37:55,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,260 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,260 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=455, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075253) roll requested 2024-11-11T16:37:55,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075261 2024-11-11T16:37:55,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075261 2024-11-11T16:37:55,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,274 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,274 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=456, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,275 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075261) roll requested 2024-11-11T16:37:55,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075275 2024-11-11T16:37:55,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075275 2024-11-11T16:37:55,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,282 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,282 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=457, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075275) roll requested 2024-11-11T16:37:55,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075282 2024-11-11T16:37:55,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075275 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075282 2024-11-11T16:37:55,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,289 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,289 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=458, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,289 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075282) roll requested 2024-11-11T16:37:55,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075290 2024-11-11T16:37:55,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075290 2024-11-11T16:37:55,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,297 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,297 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=459, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075290) roll requested 2024-11-11T16:37:55,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075297 2024-11-11T16:37:55,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075290 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075297 2024-11-11T16:37:55,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,304 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=460, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075297) roll requested 2024-11-11T16:37:55,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075305 2024-11-11T16:37:55,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075305 2024-11-11T16:37:55,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=461, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075305) roll requested 2024-11-11T16:37:55,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075312 2024-11-11T16:37:55,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075312 2024-11-11T16:37:55,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,319 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=462, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,319 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,319 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075312) roll requested 2024-11-11T16:37:55,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075320 2024-11-11T16:37:55,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075320 2024-11-11T16:37:55,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,327 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,327 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=463, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075320) roll requested 2024-11-11T16:37:55,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075328 2024-11-11T16:37:55,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075328 2024-11-11T16:37:55,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,335 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,335 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=464, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,335 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075328) roll requested 2024-11-11T16:37:55,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075335 2024-11-11T16:37:55,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075328 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075335 2024-11-11T16:37:55,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,342 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=465, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075335) roll requested 2024-11-11T16:37:55,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075342 2024-11-11T16:37:55,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075335 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075342 2024-11-11T16:37:55,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=466, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075342) roll requested 2024-11-11T16:37:55,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075349 2024-11-11T16:37:55,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075349 2024-11-11T16:37:55,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,356 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,356 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=467, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,356 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075349) roll requested 2024-11-11T16:37:55,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075357 2024-11-11T16:37:55,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075357 2024-11-11T16:37:55,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=468, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075357) roll requested 2024-11-11T16:37:55,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075364 2024-11-11T16:37:55,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075364 2024-11-11T16:37:55,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,371 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,371 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=469, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075364) roll requested 2024-11-11T16:37:55,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075371 2024-11-11T16:37:55,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075371 2024-11-11T16:37:55,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,378 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,378 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=470, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,378 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075371) roll requested 2024-11-11T16:37:55,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075379 2024-11-11T16:37:55,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075379 2024-11-11T16:37:55,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,389 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=471, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075379) roll requested 2024-11-11T16:37:55,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075390 2024-11-11T16:37:55,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075379 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075390 2024-11-11T16:37:55,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,403 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,404 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=472, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,404 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075390) roll requested 2024-11-11T16:37:55,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075404 2024-11-11T16:37:55,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075404 2024-11-11T16:37:55,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,412 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,412 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=473, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075404) roll requested 2024-11-11T16:37:55,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075412 2024-11-11T16:37:55,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075412 2024-11-11T16:37:55,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,419 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,419 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=474, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,419 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075412) roll requested 2024-11-11T16:37:55,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075419 2024-11-11T16:37:55,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075412 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075419 2024-11-11T16:37:55,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,425 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,425 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=475, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,425 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075419) roll requested 2024-11-11T16:37:55,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075425 2024-11-11T16:37:55,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075419 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075425 2024-11-11T16:37:55,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=476, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,432 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075425) roll requested 2024-11-11T16:37:55,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075432 2024-11-11T16:37:55,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075425 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075432 2024-11-11T16:37:55,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,438 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,438 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=477, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,438 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075432) roll requested 2024-11-11T16:37:55,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075438 2024-11-11T16:37:55,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075432 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075438 2024-11-11T16:37:55,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,445 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,445 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=478, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,445 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075438) roll requested 2024-11-11T16:37:55,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075445 2024-11-11T16:37:55,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075438 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075445 2024-11-11T16:37:55,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,456 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,456 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=479, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,456 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075445) roll requested 2024-11-11T16:37:55,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075456 2024-11-11T16:37:55,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075456 2024-11-11T16:37:55,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=480, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075456) roll requested 2024-11-11T16:37:55,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075465 2024-11-11T16:37:55,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075465 2024-11-11T16:37:55,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,472 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=481, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,472 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075465) roll requested 2024-11-11T16:37:55,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075473 2024-11-11T16:37:55,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075473 2024-11-11T16:37:55,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,479 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=482, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,479 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075473) roll requested 2024-11-11T16:37:55,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075479 2024-11-11T16:37:55,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075479 2024-11-11T16:37:55,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=483, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,486 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075479) roll requested 2024-11-11T16:37:55,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075486 2024-11-11T16:37:55,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075486 2024-11-11T16:37:55,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=484, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075486) roll requested 2024-11-11T16:37:55,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075492 2024-11-11T16:37:55,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075486 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075492 2024-11-11T16:37:55,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,499 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,499 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=485, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,499 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075492) roll requested 2024-11-11T16:37:55,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075499 2024-11-11T16:37:55,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075499 2024-11-11T16:37:55,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,505 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,505 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=486, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075499) roll requested 2024-11-11T16:37:55,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075505 2024-11-11T16:37:55,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075505 2024-11-11T16:37:55,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=487, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075505) roll requested 2024-11-11T16:37:55,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075512 2024-11-11T16:37:55,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075512 2024-11-11T16:37:55,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=488, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075512) roll requested 2024-11-11T16:37:55,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075524 2024-11-11T16:37:55,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075512 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075524 2024-11-11T16:37:55,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,530 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=489, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,530 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075524) roll requested 2024-11-11T16:37:55,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075530 2024-11-11T16:37:55,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075524 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075530 2024-11-11T16:37:55,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,536 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,536 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=490, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,536 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075530) roll requested 2024-11-11T16:37:55,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075536 2024-11-11T16:37:55,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075530 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075536 2024-11-11T16:37:55,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=491, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075536) roll requested 2024-11-11T16:37:55,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075543 2024-11-11T16:37:55,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075543 2024-11-11T16:37:55,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,549 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=492, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,549 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075543) roll requested 2024-11-11T16:37:55,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075549 2024-11-11T16:37:55,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075549 2024-11-11T16:37:55,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=493, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075549) roll requested 2024-11-11T16:37:55,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075557 2024-11-11T16:37:55,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075549 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075557 2024-11-11T16:37:55,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,565 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,565 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=494, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,565 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075557) roll requested 2024-11-11T16:37:55,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075565 2024-11-11T16:37:55,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075565 2024-11-11T16:37:55,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,573 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=495, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075565) roll requested 2024-11-11T16:37:55,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075573 2024-11-11T16:37:55,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075573 2024-11-11T16:37:55,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,580 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=496, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,580 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075573) roll requested 2024-11-11T16:37:55,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075580 2024-11-11T16:37:55,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075580 2024-11-11T16:37:55,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,586 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=497, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,586 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,587 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075580) roll requested 2024-11-11T16:37:55,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075587 2024-11-11T16:37:55,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075580 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075587 2024-11-11T16:37:55,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=498, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075587) roll requested 2024-11-11T16:37:55,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075593 2024-11-11T16:37:55,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075593 2024-11-11T16:37:55,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,599 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=499, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,599 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075593) roll requested 2024-11-11T16:37:55,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075600 2024-11-11T16:37:55,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075600 2024-11-11T16:37:55,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=500, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075600) roll requested 2024-11-11T16:37:55,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075608 2024-11-11T16:37:55,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075600 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075608 2024-11-11T16:37:55,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,614 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,614 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=501, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,614 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075608) roll requested 2024-11-11T16:37:55,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075614 2024-11-11T16:37:55,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075614 2024-11-11T16:37:55,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,622 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,622 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=502, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,623 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075614) roll requested 2024-11-11T16:37:55,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075623 2024-11-11T16:37:55,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075614 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075623 2024-11-11T16:37:55,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,630 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=503, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,630 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075623) roll requested 2024-11-11T16:37:55,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075630 2024-11-11T16:37:55,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075623 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075630 2024-11-11T16:37:55,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,637 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=504, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,637 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075630) roll requested 2024-11-11T16:37:55,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075638 2024-11-11T16:37:55,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075638 2024-11-11T16:37:55,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=505, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075638) roll requested 2024-11-11T16:37:55,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075645 2024-11-11T16:37:55,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075645 2024-11-11T16:37:55,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=506, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075645) roll requested 2024-11-11T16:37:55,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075652 2024-11-11T16:37:55,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075652 2024-11-11T16:37:55,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,665 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=507, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,665 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075652) roll requested 2024-11-11T16:37:55,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075665 2024-11-11T16:37:55,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075665 2024-11-11T16:37:55,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=508, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075665) roll requested 2024-11-11T16:37:55,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075674 2024-11-11T16:37:55,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075674 2024-11-11T16:37:55,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,682 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,682 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=509, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,682 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075674) roll requested 2024-11-11T16:37:55,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075683 2024-11-11T16:37:55,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075683 2024-11-11T16:37:55,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,690 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,691 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=510, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,691 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075683) roll requested 2024-11-11T16:37:55,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075691 2024-11-11T16:37:55,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075683 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075691 2024-11-11T16:37:55,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,700 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=511, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,700 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075691) roll requested 2024-11-11T16:37:55,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075701 2024-11-11T16:37:55,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075701 2024-11-11T16:37:55,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,712 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,712 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=512, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,713 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075701) roll requested 2024-11-11T16:37:55,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075713 2024-11-11T16:37:55,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075713 2024-11-11T16:37:55,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,723 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,723 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=513, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,723 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075713) roll requested 2024-11-11T16:37:55,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075723 2024-11-11T16:37:55,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075713 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075723 2024-11-11T16:37:55,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,731 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=514, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075723) roll requested 2024-11-11T16:37:55,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075732 2024-11-11T16:37:55,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075723 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075732 2024-11-11T16:37:55,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,739 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=515, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,739 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075732) roll requested 2024-11-11T16:37:55,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075739 2024-11-11T16:37:55,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075732 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075739 2024-11-11T16:37:55,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,746 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=516, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,746 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075739) roll requested 2024-11-11T16:37:55,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075746 2024-11-11T16:37:55,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075739 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075746 2024-11-11T16:37:55,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,754 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,754 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=517, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,754 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075746) roll requested 2024-11-11T16:37:55,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075754 2024-11-11T16:37:55,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075754 2024-11-11T16:37:55,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,763 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=518, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,763 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075754) roll requested 2024-11-11T16:37:55,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075763 2024-11-11T16:37:55,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075754 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075763 2024-11-11T16:37:55,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,769 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,770 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=519, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,770 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075763) roll requested 2024-11-11T16:37:55,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075770 2024-11-11T16:37:55,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075763 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075770 2024-11-11T16:37:55,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,778 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=520, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,778 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075770) roll requested 2024-11-11T16:37:55,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075778 2024-11-11T16:37:55,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075770 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075778 2024-11-11T16:37:55,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=521, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075778) roll requested 2024-11-11T16:37:55,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075785 2024-11-11T16:37:55,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075778 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075785 2024-11-11T16:37:55,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=522, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075785) roll requested 2024-11-11T16:37:55,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075791 2024-11-11T16:37:55,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075791 2024-11-11T16:37:55,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,798 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=523, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075791) roll requested 2024-11-11T16:37:55,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075799 2024-11-11T16:37:55,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075799 2024-11-11T16:37:55,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,805 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=524, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075799) roll requested 2024-11-11T16:37:55,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075806 2024-11-11T16:37:55,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075806 2024-11-11T16:37:55,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075799 is not closed yet, will try archiving it next time 2024-11-11T16:37:55,816 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=525, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,817 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075806) roll requested 2024-11-11T16:37:55,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075817 2024-11-11T16:37:55,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075817 2024-11-11T16:37:55,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,826 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,826 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=526, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,827 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075817) roll requested 2024-11-11T16:37:55,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075827 2024-11-11T16:37:55,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075817 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075827 2024-11-11T16:37:55,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,838 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,838 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=527, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,838 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075827) roll requested 2024-11-11T16:37:55,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075838 2024-11-11T16:37:55,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075827 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075838 2024-11-11T16:37:55,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=528, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075838) roll requested 2024-11-11T16:37:55,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075849 2024-11-11T16:37:55,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075838 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075849 2024-11-11T16:37:55,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=529, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075849) roll requested 2024-11-11T16:37:55,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075857 2024-11-11T16:37:55,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075857 2024-11-11T16:37:55,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=530, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075857) roll requested 2024-11-11T16:37:55,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075865 2024-11-11T16:37:55,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075857 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075865 2024-11-11T16:37:55,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,872 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=531, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075865) roll requested 2024-11-11T16:37:55,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075872 2024-11-11T16:37:55,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075872 2024-11-11T16:37:55,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=532, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075872) roll requested 2024-11-11T16:37:55,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075880 2024-11-11T16:37:55,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075880 2024-11-11T16:37:55,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=533, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,887 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075880) roll requested 2024-11-11T16:37:55,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075887 2024-11-11T16:37:55,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075887 2024-11-11T16:37:55,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,894 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,894 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=534, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075887) roll requested 2024-11-11T16:37:55,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075894 2024-11-11T16:37:55,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075894 2024-11-11T16:37:55,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=535, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,901 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075894) roll requested 2024-11-11T16:37:55,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075901 2024-11-11T16:37:55,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075901 2024-11-11T16:37:55,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,908 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,908 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=536, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,908 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075901) roll requested 2024-11-11T16:37:55,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075908 2024-11-11T16:37:55,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075908 2024-11-11T16:37:55,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,914 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=537, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075908) roll requested 2024-11-11T16:37:55,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075915 2024-11-11T16:37:55,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075908 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075915 2024-11-11T16:37:55,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,922 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,922 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=538, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,922 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075915) roll requested 2024-11-11T16:37:55,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075922 2024-11-11T16:37:55,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075922 2024-11-11T16:37:55,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,930 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,930 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=539, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075922) roll requested 2024-11-11T16:37:55,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075931 2024-11-11T16:37:55,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075922 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075931 2024-11-11T16:37:55,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,940 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=540, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,940 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075931) roll requested 2024-11-11T16:37:55,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075940 2024-11-11T16:37:55,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075940 2024-11-11T16:37:55,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,949 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=541, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075940) roll requested 2024-11-11T16:37:55,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075949 2024-11-11T16:37:55,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075940 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075949 2024-11-11T16:37:55,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,957 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,957 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=542, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,957 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075949) roll requested 2024-11-11T16:37:55,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075957 2024-11-11T16:37:55,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075957 2024-11-11T16:37:55,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,968 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,969 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=543, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,969 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075957) roll requested 2024-11-11T16:37:55,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075969 2024-11-11T16:37:55,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075957 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075969 2024-11-11T16:37:55,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,978 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=544, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,978 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,979 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075969) roll requested 2024-11-11T16:37:55,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075979 2024-11-11T16:37:55,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:55,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075969 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075979 2024-11-11T16:37:55,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:55,989 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:55,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=545, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:55,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:55,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075979) roll requested 2024-11-11T16:37:55,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343075989 2024-11-11T16:37:56,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075989 2024-11-11T16:37:56,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,003 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=546, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,004 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343075989) roll requested 2024-11-11T16:37:56,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076004 2024-11-11T16:37:56,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343075989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076004 2024-11-11T16:37:56,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=547, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076004) roll requested 2024-11-11T16:37:56,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076013 2024-11-11T16:37:56,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076004 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076013 2024-11-11T16:37:56,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=548, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076013) roll requested 2024-11-11T16:37:56,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076021 2024-11-11T16:37:56,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076021 2024-11-11T16:37:56,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=549, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,029 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076021) roll requested 2024-11-11T16:37:56,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076030 2024-11-11T16:37:56,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076030 2024-11-11T16:37:56,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=550, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076030) roll requested 2024-11-11T16:37:56,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076054 2024-11-11T16:37:56,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076030 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076054 2024-11-11T16:37:56,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,074 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,074 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=551, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,075 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076054) roll requested 2024-11-11T16:37:56,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076075 2024-11-11T16:37:56,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076075 2024-11-11T16:37:56,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,129 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=552, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,129 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,132 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076075) roll requested 2024-11-11T16:37:56,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076133 2024-11-11T16:37:56,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076075 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076133 2024-11-11T16:37:56,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,157 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=553, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076133) roll requested 2024-11-11T16:37:56,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076158 2024-11-11T16:37:56,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076158 2024-11-11T16:37:56,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=554, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076158) roll requested 2024-11-11T16:37:56,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076174 2024-11-11T16:37:56,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076158 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076174 2024-11-11T16:37:56,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,186 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,187 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=555, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,187 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076174) roll requested 2024-11-11T16:37:56,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076187 2024-11-11T16:37:56,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076187 2024-11-11T16:37:56,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,200 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=556, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,200 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,200 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076187) roll requested 2024-11-11T16:37:56,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076201 2024-11-11T16:37:56,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076187 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076201 2024-11-11T16:37:56,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=557, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076201) roll requested 2024-11-11T16:37:56,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076217 2024-11-11T16:37:56,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076217 2024-11-11T16:37:56,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,226 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=558, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076217) roll requested 2024-11-11T16:37:56,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076226 2024-11-11T16:37:56,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076226 2024-11-11T16:37:56,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,241 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,241 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=559, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076226) roll requested 2024-11-11T16:37:56,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076241 2024-11-11T16:37:56,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076241 2024-11-11T16:37:56,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,250 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,250 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=560, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,250 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076241) roll requested 2024-11-11T16:37:56,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076250 2024-11-11T16:37:56,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076250 2024-11-11T16:37:56,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,259 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,259 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=561, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,259 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076250) roll requested 2024-11-11T16:37:56,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076259 2024-11-11T16:37:56,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076250 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076259 2024-11-11T16:37:56,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=562, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076259) roll requested 2024-11-11T16:37:56,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076269 2024-11-11T16:37:56,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076259 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076269 2024-11-11T16:37:56,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=563, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076269) roll requested 2024-11-11T16:37:56,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076277 2024-11-11T16:37:56,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076277 2024-11-11T16:37:56,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,318 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=564, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076277) roll requested 2024-11-11T16:37:56,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076319 2024-11-11T16:37:56,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076319 2024-11-11T16:37:56,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,345 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,345 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=565, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,345 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076319) roll requested 2024-11-11T16:37:56,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076345 2024-11-11T16:37:56,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076319 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076345 2024-11-11T16:37:56,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,361 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,361 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=566, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076345) roll requested 2024-11-11T16:37:56,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076361 2024-11-11T16:37:56,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076361 2024-11-11T16:37:56,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,372 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,373 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=567, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076361) roll requested 2024-11-11T16:37:56,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076373 2024-11-11T16:37:56,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076373 2024-11-11T16:37:56,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=568, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076373) roll requested 2024-11-11T16:37:56,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076381 2024-11-11T16:37:56,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076381 2024-11-11T16:37:56,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,395 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=569, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,395 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076381) roll requested 2024-11-11T16:37:56,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076395 2024-11-11T16:37:56,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076395 2024-11-11T16:37:56,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,403 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,403 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=570, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,404 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076395) roll requested 2024-11-11T16:37:56,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076404 2024-11-11T16:37:56,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076395 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076404 2024-11-11T16:37:56,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,411 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=571, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,411 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076404) roll requested 2024-11-11T16:37:56,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076411 2024-11-11T16:37:56,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076411 2024-11-11T16:37:56,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,419 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,419 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=572, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,419 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076411) roll requested 2024-11-11T16:37:56,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076419 2024-11-11T16:37:56,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076411 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076419 2024-11-11T16:37:56,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=573, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076419) roll requested 2024-11-11T16:37:56,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076428 2024-11-11T16:37:56,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076419 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076428 2024-11-11T16:37:56,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,437 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=574, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,437 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076428) roll requested 2024-11-11T16:37:56,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076437 2024-11-11T16:37:56,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076437 2024-11-11T16:37:56,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,453 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,453 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=575, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,454 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076437) roll requested 2024-11-11T16:37:56,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076454 2024-11-11T16:37:56,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076437 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076454 2024-11-11T16:37:56,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=576, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076454) roll requested 2024-11-11T16:37:56,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076486 2024-11-11T16:37:56,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076454 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076486 2024-11-11T16:37:56,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=577, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,499 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076486) roll requested 2024-11-11T16:37:56,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076499 2024-11-11T16:37:56,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076486 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076499 2024-11-11T16:37:56,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,530 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=578, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,530 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076499) roll requested 2024-11-11T16:37:56,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076530 2024-11-11T16:37:56,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076530 2024-11-11T16:37:56,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=579, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076530) roll requested 2024-11-11T16:37:56,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076557 2024-11-11T16:37:56,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076530 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076557 2024-11-11T16:37:56,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,573 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,574 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=580, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,574 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076557) roll requested 2024-11-11T16:37:56,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076574 2024-11-11T16:37:56,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076574 2024-11-11T16:37:56,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=581, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076574) roll requested 2024-11-11T16:37:56,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076593 2024-11-11T16:37:56,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076574 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076593 2024-11-11T16:37:56,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,602 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=582, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,602 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076593) roll requested 2024-11-11T16:37:56,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076602 2024-11-11T16:37:56,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076602 2024-11-11T16:37:56,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,617 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=583, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,617 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076602) roll requested 2024-11-11T16:37:56,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076617 2024-11-11T16:37:56,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076602 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076617 2024-11-11T16:37:56,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,638 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,639 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=584, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,639 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076617) roll requested 2024-11-11T16:37:56,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076639 2024-11-11T16:37:56,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076639 2024-11-11T16:37:56,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,649 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,649 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=585, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,650 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076639) roll requested 2024-11-11T16:37:56,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076650 2024-11-11T16:37:56,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076639 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076650 2024-11-11T16:37:56,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,665 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=586, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,665 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076650) roll requested 2024-11-11T16:37:56,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076666 2024-11-11T16:37:56,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076650 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076666 2024-11-11T16:37:56,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,685 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,685 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=587, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,685 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076666) roll requested 2024-11-11T16:37:56,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076685 2024-11-11T16:37:56,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076666 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076685 2024-11-11T16:37:56,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,699 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,699 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=588, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,699 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076685) roll requested 2024-11-11T16:37:56,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076699 2024-11-11T16:37:56,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076699 2024-11-11T16:37:56,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,712 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,712 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=589, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,712 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076699) roll requested 2024-11-11T16:37:56,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076712 2024-11-11T16:37:56,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076699 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076712 2024-11-11T16:37:56,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,729 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=590, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076712) roll requested 2024-11-11T16:37:56,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076730 2024-11-11T16:37:56,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076712 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076730 2024-11-11T16:37:56,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,747 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=591, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,747 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076730) roll requested 2024-11-11T16:37:56,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076747 2024-11-11T16:37:56,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076747 2024-11-11T16:37:56,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=592, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076747) roll requested 2024-11-11T16:37:56,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076767 2024-11-11T16:37:56,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076747 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076767 2024-11-11T16:37:56,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,783 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=593, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076767) roll requested 2024-11-11T16:37:56,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076784 2024-11-11T16:37:56,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076784 2024-11-11T16:37:56,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=594, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076784) roll requested 2024-11-11T16:37:56,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076802 2024-11-11T16:37:56,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076802 2024-11-11T16:37:56,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,810 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=595, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,810 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,811 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076802) roll requested 2024-11-11T16:37:56,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076812 2024-11-11T16:37:56,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076812 2024-11-11T16:37:56,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=596, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,820 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076812) roll requested 2024-11-11T16:37:56,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076821 2024-11-11T16:37:56,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076812 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076821 2024-11-11T16:37:56,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=597, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,829 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076821) roll requested 2024-11-11T16:37:56,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076829 2024-11-11T16:37:56,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076829 2024-11-11T16:37:56,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=598, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076829) roll requested 2024-11-11T16:37:56,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076847 2024-11-11T16:37:56,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076847 2024-11-11T16:37:56,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,859 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,859 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=599, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,859 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076847) roll requested 2024-11-11T16:37:56,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076859 2024-11-11T16:37:56,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076859 2024-11-11T16:37:56,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,869 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,869 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=600, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,869 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076859) roll requested 2024-11-11T16:37:56,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076869 2024-11-11T16:37:56,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076859 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076869 2024-11-11T16:37:56,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,876 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,876 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=601, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,876 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076869) roll requested 2024-11-11T16:37:56,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076876 2024-11-11T16:37:56,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076869 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076876 2024-11-11T16:37:56,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,883 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,883 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=602, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,883 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076876) roll requested 2024-11-11T16:37:56,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076883 2024-11-11T16:37:56,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076876 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076883 2024-11-11T16:37:56,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=603, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076883) roll requested 2024-11-11T16:37:56,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076890 2024-11-11T16:37:56,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076883 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076890 2024-11-11T16:37:56,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,898 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,898 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=604, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,898 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076890) roll requested 2024-11-11T16:37:56,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076898 2024-11-11T16:37:56,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076898 2024-11-11T16:37:56,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,905 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=605, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,905 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076898) roll requested 2024-11-11T16:37:56,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076905 2024-11-11T16:37:56,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076898 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076905 2024-11-11T16:37:56,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=606, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076905) roll requested 2024-11-11T16:37:56,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076913 2024-11-11T16:37:56,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076905 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076913 2024-11-11T16:37:56,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=607, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076913) roll requested 2024-11-11T16:37:56,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076920 2024-11-11T16:37:56,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076920 2024-11-11T16:37:56,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,928 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=608, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,928 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076920) roll requested 2024-11-11T16:37:56,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076928 2024-11-11T16:37:56,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076928 2024-11-11T16:37:56,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=609, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076928) roll requested 2024-11-11T16:37:56,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076936 2024-11-11T16:37:56,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076936 2024-11-11T16:37:56,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,943 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=610, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076936) roll requested 2024-11-11T16:37:56,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076944 2024-11-11T16:37:56,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076944 2024-11-11T16:37:56,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,951 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,951 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=611, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076944) roll requested 2024-11-11T16:37:56,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076952 2024-11-11T16:37:56,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076952 2024-11-11T16:37:56,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=612, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076952) roll requested 2024-11-11T16:37:56,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076959 2024-11-11T16:37:56,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076959 2024-11-11T16:37:56,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,969 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,969 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=613, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,969 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076959) roll requested 2024-11-11T16:37:56,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076969 2024-11-11T16:37:56,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076969 2024-11-11T16:37:56,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=614, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076969) roll requested 2024-11-11T16:37:56,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076977 2024-11-11T16:37:56,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076969 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076977 2024-11-11T16:37:56,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=615, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076977) roll requested 2024-11-11T16:37:56,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076984 2024-11-11T16:37:56,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076984 2024-11-11T16:37:56,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,992 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,992 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=616, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,992 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:56,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076984) roll requested 2024-11-11T16:37:56,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343076993 2024-11-11T16:37:56,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:56,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076993 2024-11-11T16:37:56,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:56,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:56,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=617, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:56,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343076993) roll requested 2024-11-11T16:37:57,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077000 2024-11-11T16:37:57,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343076993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077000 2024-11-11T16:37:57,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,007 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,008 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=618, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,008 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077000) roll requested 2024-11-11T16:37:57,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077008 2024-11-11T16:37:57,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077008 2024-11-11T16:37:57,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,015 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,015 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=619, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,015 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077008) roll requested 2024-11-11T16:37:57,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077015 2024-11-11T16:37:57,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077008 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077015 2024-11-11T16:37:57,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,022 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,023 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=620, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,023 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077015) roll requested 2024-11-11T16:37:57,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077023 2024-11-11T16:37:57,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077023 2024-11-11T16:37:57,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,030 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,030 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=621, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,030 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077023) roll requested 2024-11-11T16:37:57,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077030 2024-11-11T16:37:57,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077023 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077030 2024-11-11T16:37:57,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=622, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,038 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077030) roll requested 2024-11-11T16:37:57,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077038 2024-11-11T16:37:57,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077030 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077038 2024-11-11T16:37:57,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=623, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,044 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077038) roll requested 2024-11-11T16:37:57,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077045 2024-11-11T16:37:57,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077045 2024-11-11T16:37:57,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,052 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,052 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=624, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,053 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077045) roll requested 2024-11-11T16:37:57,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077053 2024-11-11T16:37:57,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077053 2024-11-11T16:37:57,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,059 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=625, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,059 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077053) roll requested 2024-11-11T16:37:57,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077059 2024-11-11T16:37:57,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077053 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077059 2024-11-11T16:37:57,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,066 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=626, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,066 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077059) roll requested 2024-11-11T16:37:57,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077066 2024-11-11T16:37:57,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077066 2024-11-11T16:37:57,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=627, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,072 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077066) roll requested 2024-11-11T16:37:57,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077073 2024-11-11T16:37:57,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077073 2024-11-11T16:37:57,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=628, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077073) roll requested 2024-11-11T16:37:57,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077079 2024-11-11T16:37:57,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077079 2024-11-11T16:37:57,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=629, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077079) roll requested 2024-11-11T16:37:57,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077086 2024-11-11T16:37:57,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077086 2024-11-11T16:37:57,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,095 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,096 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=630, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,096 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077086) roll requested 2024-11-11T16:37:57,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077096 2024-11-11T16:37:57,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077096 2024-11-11T16:37:57,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=631, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,113 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077096) roll requested 2024-11-11T16:37:57,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077113 2024-11-11T16:37:57,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077096 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077113 2024-11-11T16:37:57,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=632, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077113) roll requested 2024-11-11T16:37:57,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077121 2024-11-11T16:37:57,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077121 2024-11-11T16:37:57,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,128 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=633, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,129 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077121) roll requested 2024-11-11T16:37:57,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077129 2024-11-11T16:37:57,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077129 2024-11-11T16:37:57,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,135 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=634, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,135 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077129) roll requested 2024-11-11T16:37:57,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077135 2024-11-11T16:37:57,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077129 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077135 2024-11-11T16:37:57,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,143 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=635, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,143 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077135) roll requested 2024-11-11T16:37:57,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077143 2024-11-11T16:37:57,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077135 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077143 2024-11-11T16:37:57,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,150 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,151 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=636, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,151 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077143) roll requested 2024-11-11T16:37:57,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077151 2024-11-11T16:37:57,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077143 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077151 2024-11-11T16:37:57,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=637, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077151) roll requested 2024-11-11T16:37:57,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077159 2024-11-11T16:37:57,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077151 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077159 2024-11-11T16:37:57,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,166 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,166 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=638, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077159) roll requested 2024-11-11T16:37:57,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077166 2024-11-11T16:37:57,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077159 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077166 2024-11-11T16:37:57,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,174 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,174 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=639, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,174 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077166) roll requested 2024-11-11T16:37:57,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077174 2024-11-11T16:37:57,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077174 2024-11-11T16:37:57,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=640, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077174) roll requested 2024-11-11T16:37:57,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077184 2024-11-11T16:37:57,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077184 2024-11-11T16:37:57,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=641, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077184) roll requested 2024-11-11T16:37:57,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077197 2024-11-11T16:37:57,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077197 2024-11-11T16:37:57,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=642, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,205 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077197) roll requested 2024-11-11T16:37:57,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077206 2024-11-11T16:37:57,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077206 2024-11-11T16:37:57,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,213 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,213 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=643, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,213 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077206) roll requested 2024-11-11T16:37:57,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077213 2024-11-11T16:37:57,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077206 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077213 2024-11-11T16:37:57,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,221 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,221 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=644, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,221 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077213) roll requested 2024-11-11T16:37:57,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077221 2024-11-11T16:37:57,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077213 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077221 2024-11-11T16:37:57,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,228 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=645, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077221) roll requested 2024-11-11T16:37:57,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077229 2024-11-11T16:37:57,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077221 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077229 2024-11-11T16:37:57,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,237 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=646, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077229) roll requested 2024-11-11T16:37:57,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077237 2024-11-11T16:37:57,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077237 2024-11-11T16:37:57,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=647, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077237) roll requested 2024-11-11T16:37:57,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077245 2024-11-11T16:37:57,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077245 2024-11-11T16:37:57,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=648, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,256 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077245) roll requested 2024-11-11T16:37:57,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077257 2024-11-11T16:37:57,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077257 2024-11-11T16:37:57,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,267 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,267 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=649, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077257) roll requested 2024-11-11T16:37:57,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077268 2024-11-11T16:37:57,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077268 2024-11-11T16:37:57,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=650, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077268) roll requested 2024-11-11T16:37:57,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077303 2024-11-11T16:37:57,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077303 2024-11-11T16:37:57,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,315 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=651, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,315 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,317 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077303) roll requested 2024-11-11T16:37:57,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077317 2024-11-11T16:37:57,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077317 2024-11-11T16:37:57,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=652, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077317) roll requested 2024-11-11T16:37:57,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077329 2024-11-11T16:37:57,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077329 2024-11-11T16:37:57,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,337 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,337 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=653, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077329) roll requested 2024-11-11T16:37:57,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077337 2024-11-11T16:37:57,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077337 2024-11-11T16:37:57,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,345 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,345 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=654, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,345 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077337) roll requested 2024-11-11T16:37:57,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077345 2024-11-11T16:37:57,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077345 2024-11-11T16:37:57,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,353 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,353 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=655, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,353 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077345) roll requested 2024-11-11T16:37:57,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077353 2024-11-11T16:37:57,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077353 2024-11-11T16:37:57,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,361 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,361 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=656, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077353) roll requested 2024-11-11T16:37:57,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077361 2024-11-11T16:37:57,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077353 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077361 2024-11-11T16:37:57,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=657, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,369 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077361) roll requested 2024-11-11T16:37:57,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077369 2024-11-11T16:37:57,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077369 2024-11-11T16:37:57,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=658, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077369) roll requested 2024-11-11T16:37:57,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077381 2024-11-11T16:37:57,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077381 2024-11-11T16:37:57,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=659, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,390 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077381) roll requested 2024-11-11T16:37:57,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077390 2024-11-11T16:37:57,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077390 2024-11-11T16:37:57,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,399 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,399 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=660, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,399 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077390) roll requested 2024-11-11T16:37:57,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077399 2024-11-11T16:37:57,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077399 2024-11-11T16:37:57,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,407 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=661, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077399) roll requested 2024-11-11T16:37:57,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077408 2024-11-11T16:37:57,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077399 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077408 2024-11-11T16:37:57,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,416 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=662, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077408) roll requested 2024-11-11T16:37:57,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077417 2024-11-11T16:37:57,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077417 2024-11-11T16:37:57,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=663, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077417) roll requested 2024-11-11T16:37:57,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077424 2024-11-11T16:37:57,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077424 2024-11-11T16:37:57,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,431 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=664, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077424) roll requested 2024-11-11T16:37:57,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077431 2024-11-11T16:37:57,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077431 2024-11-11T16:37:57,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,438 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,438 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=665, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,438 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077431) roll requested 2024-11-11T16:37:57,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077438 2024-11-11T16:37:57,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077438 2024-11-11T16:37:57,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,444 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,444 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=666, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,445 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077438) roll requested 2024-11-11T16:37:57,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077445 2024-11-11T16:37:57,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077438 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077445 2024-11-11T16:37:57,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,451 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,451 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=667, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,451 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077445) roll requested 2024-11-11T16:37:57,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077451 2024-11-11T16:37:57,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077451 2024-11-11T16:37:57,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,458 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=668, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,458 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077451) roll requested 2024-11-11T16:37:57,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077458 2024-11-11T16:37:57,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077451 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077458 2024-11-11T16:37:57,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=669, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077458) roll requested 2024-11-11T16:37:57,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077465 2024-11-11T16:37:57,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077458 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077465 2024-11-11T16:37:57,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,471 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,471 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=670, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,471 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077465) roll requested 2024-11-11T16:37:57,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077471 2024-11-11T16:37:57,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077471 2024-11-11T16:37:57,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=671, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077471) roll requested 2024-11-11T16:37:57,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077478 2024-11-11T16:37:57,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077471 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077478 2024-11-11T16:37:57,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=672, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077478) roll requested 2024-11-11T16:37:57,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077484 2024-11-11T16:37:57,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077484 2024-11-11T16:37:57,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,491 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=673, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,491 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077484) roll requested 2024-11-11T16:37:57,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077491 2024-11-11T16:37:57,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077491 2024-11-11T16:37:57,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=674, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077491) roll requested 2024-11-11T16:37:57,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077498 2024-11-11T16:37:57,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077498 2024-11-11T16:37:57,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=675, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077498) roll requested 2024-11-11T16:37:57,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077504 2024-11-11T16:37:57,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077504 2024-11-11T16:37:57,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,510 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,510 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=676, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,510 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077504) roll requested 2024-11-11T16:37:57,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077511 2024-11-11T16:37:57,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077511 2024-11-11T16:37:57,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,517 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,517 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=677, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,517 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077511) roll requested 2024-11-11T16:37:57,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077517 2024-11-11T16:37:57,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077517 2024-11-11T16:37:57,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=678, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077517) roll requested 2024-11-11T16:37:57,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077524 2024-11-11T16:37:57,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077517 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077524 2024-11-11T16:37:57,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,530 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=679, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,530 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077524) roll requested 2024-11-11T16:37:57,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077530 2024-11-11T16:37:57,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077524 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077530 2024-11-11T16:37:57,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,536 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=680, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077530) roll requested 2024-11-11T16:37:57,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077537 2024-11-11T16:37:57,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077530 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077537 2024-11-11T16:37:57,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=681, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077537) roll requested 2024-11-11T16:37:57,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077543 2024-11-11T16:37:57,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077543 2024-11-11T16:37:57,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=682, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077543) roll requested 2024-11-11T16:37:57,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077550 2024-11-11T16:37:57,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077550 2024-11-11T16:37:57,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,556 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=683, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,556 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077550) roll requested 2024-11-11T16:37:57,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077556 2024-11-11T16:37:57,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077556 2024-11-11T16:37:57,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=684, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077556) roll requested 2024-11-11T16:37:57,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077564 2024-11-11T16:37:57,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077556 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077564 2024-11-11T16:37:57,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,573 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=685, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077564) roll requested 2024-11-11T16:37:57,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077573 2024-11-11T16:37:57,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077573 2024-11-11T16:37:57,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,580 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=686, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,580 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077573) roll requested 2024-11-11T16:37:57,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077580 2024-11-11T16:37:57,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077580 2024-11-11T16:37:57,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,588 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,588 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=687, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,588 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077580) roll requested 2024-11-11T16:37:57,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077588 2024-11-11T16:37:57,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077580 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077588 2024-11-11T16:37:57,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,596 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=688, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,596 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077588) roll requested 2024-11-11T16:37:57,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077596 2024-11-11T16:37:57,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077588 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077596 2024-11-11T16:37:57,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,604 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=689, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077596) roll requested 2024-11-11T16:37:57,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077604 2024-11-11T16:37:57,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077604 2024-11-11T16:37:57,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=690, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077604) roll requested 2024-11-11T16:37:57,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077621 2024-11-11T16:37:57,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077621 2024-11-11T16:37:57,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,630 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=691, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,630 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077621) roll requested 2024-11-11T16:37:57,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077630 2024-11-11T16:37:57,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077630 2024-11-11T16:37:57,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,640 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,641 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=692, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,641 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077630) roll requested 2024-11-11T16:37:57,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077641 2024-11-11T16:37:57,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077641 2024-11-11T16:37:57,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,648 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=693, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,648 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077641) roll requested 2024-11-11T16:37:57,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077648 2024-11-11T16:37:57,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077641 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077648 2024-11-11T16:37:57,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,657 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=694, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,657 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077648) roll requested 2024-11-11T16:37:57,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077657 2024-11-11T16:37:57,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077648 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077657 2024-11-11T16:37:57,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,665 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=695, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,666 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077657) roll requested 2024-11-11T16:37:57,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077666 2024-11-11T16:37:57,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077657 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077666 2024-11-11T16:37:57,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=696, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077666) roll requested 2024-11-11T16:37:57,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077674 2024-11-11T16:37:57,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077666 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077674 2024-11-11T16:37:57,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,683 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,683 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=697, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,683 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077674) roll requested 2024-11-11T16:37:57,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077683 2024-11-11T16:37:57,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077683 2024-11-11T16:37:57,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,690 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,690 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=698, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,690 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077683) roll requested 2024-11-11T16:37:57,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077691 2024-11-11T16:37:57,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077683 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077691 2024-11-11T16:37:57,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,698 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=699, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,698 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077691) roll requested 2024-11-11T16:37:57,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077698 2024-11-11T16:37:57,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077698 2024-11-11T16:37:57,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,706 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,706 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=700, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,706 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077698) roll requested 2024-11-11T16:37:57,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077706 2024-11-11T16:37:57,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077698 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077706 2024-11-11T16:37:57,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,714 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=701, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077706) roll requested 2024-11-11T16:37:57,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077715 2024-11-11T16:37:57,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077706 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077715 2024-11-11T16:37:57,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,722 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,722 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=702, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,722 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077715) roll requested 2024-11-11T16:37:57,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077722 2024-11-11T16:37:57,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077715 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077722 2024-11-11T16:37:57,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,733 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=703, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,733 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077722) roll requested 2024-11-11T16:37:57,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077733 2024-11-11T16:37:57,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077722 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077733 2024-11-11T16:37:57,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,745 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,746 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=704, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,746 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077733) roll requested 2024-11-11T16:37:57,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077746 2024-11-11T16:37:57,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077746 2024-11-11T16:37:57,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,754 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=705, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,754 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077746) roll requested 2024-11-11T16:37:57,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077754 2024-11-11T16:37:57,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077754 2024-11-11T16:37:57,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,763 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=706, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,763 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077754) roll requested 2024-11-11T16:37:57,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077763 2024-11-11T16:37:57,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077754 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077763 2024-11-11T16:37:57,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,770 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,770 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=707, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,770 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077763) roll requested 2024-11-11T16:37:57,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077771 2024-11-11T16:37:57,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077763 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077771 2024-11-11T16:37:57,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,778 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=708, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,778 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077771) roll requested 2024-11-11T16:37:57,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077778 2024-11-11T16:37:57,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077771 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077778 2024-11-11T16:37:57,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=709, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077778) roll requested 2024-11-11T16:37:57,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077791 2024-11-11T16:37:57,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077778 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077791 2024-11-11T16:37:57,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=710, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077791) roll requested 2024-11-11T16:37:57,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077799 2024-11-11T16:37:57,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077799 2024-11-11T16:37:57,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=711, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,807 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077799) roll requested 2024-11-11T16:37:57,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077807 2024-11-11T16:37:57,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077807 2024-11-11T16:37:57,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,814 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=712, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077807) roll requested 2024-11-11T16:37:57,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077815 2024-11-11T16:37:57,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077807 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077815 2024-11-11T16:37:57,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=713, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,822 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077815) roll requested 2024-11-11T16:37:57,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077822 2024-11-11T16:37:57,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077822 2024-11-11T16:37:57,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=714, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077822) roll requested 2024-11-11T16:37:57,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077830 2024-11-11T16:37:57,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077830 2024-11-11T16:37:57,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,837 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=715, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077830) roll requested 2024-11-11T16:37:57,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077837 2024-11-11T16:37:57,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077837 2024-11-11T16:37:57,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,844 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,844 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=716, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,844 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077837) roll requested 2024-11-11T16:37:57,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077844 2024-11-11T16:37:57,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077844 2024-11-11T16:37:57,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=717, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077844) roll requested 2024-11-11T16:37:57,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077856 2024-11-11T16:37:57,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077844 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077856 2024-11-11T16:37:57,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,863 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,863 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=718, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,863 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077856) roll requested 2024-11-11T16:37:57,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077863 2024-11-11T16:37:57,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077863 2024-11-11T16:37:57,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=719, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077863) roll requested 2024-11-11T16:37:57,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077873 2024-11-11T16:37:57,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077863 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077873 2024-11-11T16:37:57,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=720, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077873) roll requested 2024-11-11T16:37:57,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077880 2024-11-11T16:37:57,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077880 2024-11-11T16:37:57,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=721, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,887 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077880) roll requested 2024-11-11T16:37:57,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077887 2024-11-11T16:37:57,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077887 2024-11-11T16:37:57,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=722, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077887) roll requested 2024-11-11T16:37:57,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077893 2024-11-11T16:37:57,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077893 2024-11-11T16:37:57,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=723, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077893) roll requested 2024-11-11T16:37:57,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077900 2024-11-11T16:37:57,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077900 2024-11-11T16:37:57,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=724, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077900) roll requested 2024-11-11T16:37:57,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077906 2024-11-11T16:37:57,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077906 2024-11-11T16:37:57,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=725, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077906) roll requested 2024-11-11T16:37:57,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077913 2024-11-11T16:37:57,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077913 2024-11-11T16:37:57,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,919 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,919 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=726, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,919 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077913) roll requested 2024-11-11T16:37:57,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077919 2024-11-11T16:37:57,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077919 2024-11-11T16:37:57,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,926 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=727, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,926 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077919) roll requested 2024-11-11T16:37:57,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077926 2024-11-11T16:37:57,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077919 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077926 2024-11-11T16:37:57,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,932 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=728, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,932 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077926) roll requested 2024-11-11T16:37:57,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077932 2024-11-11T16:37:57,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077926 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077932 2024-11-11T16:37:57,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,939 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=729, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,939 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077932) roll requested 2024-11-11T16:37:57,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077939 2024-11-11T16:37:57,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077932 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077939 2024-11-11T16:37:57,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,945 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=730, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,945 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077939) roll requested 2024-11-11T16:37:57,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077945 2024-11-11T16:37:57,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077939 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077945 2024-11-11T16:37:57,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=731, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077945) roll requested 2024-11-11T16:37:57,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077952 2024-11-11T16:37:57,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077945 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077952 2024-11-11T16:37:57,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=732, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077952) roll requested 2024-11-11T16:37:57,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077959 2024-11-11T16:37:57,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077959 2024-11-11T16:37:57,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=733, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077959) roll requested 2024-11-11T16:37:57,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077966 2024-11-11T16:37:57,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077966 2024-11-11T16:37:57,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,973 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=734, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077966) roll requested 2024-11-11T16:37:57,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077973 2024-11-11T16:37:57,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077966 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077973 2024-11-11T16:37:57,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,979 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,980 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=735, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,980 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077973) roll requested 2024-11-11T16:37:57,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077980 2024-11-11T16:37:57,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077980 2024-11-11T16:37:57,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,986 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=736, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077980) roll requested 2024-11-11T16:37:57,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077987 2024-11-11T16:37:57,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077980 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077987 2024-11-11T16:37:57,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=737, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:57,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:57,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077987) roll requested 2024-11-11T16:37:57,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343077993 2024-11-11T16:37:57,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:57,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077993 2024-11-11T16:37:57,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:57,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:57,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,000 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=738, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343077993) roll requested 2024-11-11T16:37:58,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078000 2024-11-11T16:37:58,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343077993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078000 2024-11-11T16:37:58,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,006 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=739, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078000) roll requested 2024-11-11T16:37:58,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078007 2024-11-11T16:37:58,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078007 2024-11-11T16:37:58,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=740, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078007) roll requested 2024-11-11T16:37:58,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078013 2024-11-11T16:37:58,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078013 2024-11-11T16:37:58,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=741, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078013) roll requested 2024-11-11T16:37:58,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078020 2024-11-11T16:37:58,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078020 2024-11-11T16:37:58,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=742, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078020) roll requested 2024-11-11T16:37:58,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078033 2024-11-11T16:37:58,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078033 2024-11-11T16:37:58,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,039 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,039 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=743, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,039 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078033) roll requested 2024-11-11T16:37:58,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078039 2024-11-11T16:37:58,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078039 2024-11-11T16:37:58,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,046 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,046 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=744, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,046 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078039) roll requested 2024-11-11T16:37:58,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078046 2024-11-11T16:37:58,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078039 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078046 2024-11-11T16:37:58,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,052 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,053 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=745, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,053 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078046) roll requested 2024-11-11T16:37:58,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078053 2024-11-11T16:37:58,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078046 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078053 2024-11-11T16:37:58,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,059 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=746, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,060 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078053) roll requested 2024-11-11T16:37:58,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078060 2024-11-11T16:37:58,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078053 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078060 2024-11-11T16:37:58,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,066 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,066 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=747, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,066 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078060) roll requested 2024-11-11T16:37:58,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078066 2024-11-11T16:37:58,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078066 2024-11-11T16:37:58,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,073 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=748, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078066) roll requested 2024-11-11T16:37:58,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078073 2024-11-11T16:37:58,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078073 2024-11-11T16:37:58,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=749, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,080 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078073) roll requested 2024-11-11T16:37:58,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078080 2024-11-11T16:37:58,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078080 2024-11-11T16:37:58,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,089 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=750, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,090 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078080) roll requested 2024-11-11T16:37:58,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078090 2024-11-11T16:37:58,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078080 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078090 2024-11-11T16:37:58,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,098 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=751, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078090) roll requested 2024-11-11T16:37:58,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078098 2024-11-11T16:37:58,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078090 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078098 2024-11-11T16:37:58,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,108 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=752, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,108 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078098) roll requested 2024-11-11T16:37:58,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078108 2024-11-11T16:37:58,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078108 2024-11-11T16:37:58,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,115 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,115 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=753, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,115 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078108) roll requested 2024-11-11T16:37:58,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078115 2024-11-11T16:37:58,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078108 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078115 2024-11-11T16:37:58,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,124 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=754, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,124 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078115) roll requested 2024-11-11T16:37:58,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078124 2024-11-11T16:37:58,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078115 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078124 2024-11-11T16:37:58,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,131 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,132 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=755, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,132 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078124) roll requested 2024-11-11T16:37:58,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078132 2024-11-11T16:37:58,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078124 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078132 2024-11-11T16:37:58,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,140 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,140 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=756, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078132) roll requested 2024-11-11T16:37:58,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078140 2024-11-11T16:37:58,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078132 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078140 2024-11-11T16:37:58,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,163 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=757, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,163 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078140) roll requested 2024-11-11T16:37:58,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078163 2024-11-11T16:37:58,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078140 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078163 2024-11-11T16:37:58,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,172 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=758, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,172 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078163) roll requested 2024-11-11T16:37:58,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078172 2024-11-11T16:37:58,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078163 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078172 2024-11-11T16:37:58,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,189 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=759, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078172) roll requested 2024-11-11T16:37:58,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078189 2024-11-11T16:37:58,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078172 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078189 2024-11-11T16:37:58,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=760, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078189) roll requested 2024-11-11T16:37:58,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078198 2024-11-11T16:37:58,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078198 2024-11-11T16:37:58,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=761, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078198) roll requested 2024-11-11T16:37:58,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078209 2024-11-11T16:37:58,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078209 2024-11-11T16:37:58,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=762, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,219 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078209) roll requested 2024-11-11T16:37:58,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078219 2024-11-11T16:37:58,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078219 2024-11-11T16:37:58,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=763, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078219) roll requested 2024-11-11T16:37:58,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078230 2024-11-11T16:37:58,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078230 2024-11-11T16:37:58,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,238 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,238 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=764, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,238 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078230) roll requested 2024-11-11T16:37:58,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078238 2024-11-11T16:37:58,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078230 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078238 2024-11-11T16:37:58,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,254 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=765, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078238) roll requested 2024-11-11T16:37:58,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078255 2024-11-11T16:37:58,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078238 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078255 2024-11-11T16:37:58,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,267 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,267 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=766, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,267 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078255) roll requested 2024-11-11T16:37:58,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078268 2024-11-11T16:37:58,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078268 2024-11-11T16:37:58,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,274 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,274 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=767, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,274 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078268) roll requested 2024-11-11T16:37:58,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078274 2024-11-11T16:37:58,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078274 2024-11-11T16:37:58,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,281 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,282 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=768, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078274) roll requested 2024-11-11T16:37:58,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078282 2024-11-11T16:37:58,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078274 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078282 2024-11-11T16:37:58,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,289 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,289 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=769, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,289 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078282) roll requested 2024-11-11T16:37:58,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078289 2024-11-11T16:37:58,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078289 2024-11-11T16:37:58,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,295 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,295 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=770, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078289) roll requested 2024-11-11T16:37:58,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078296 2024-11-11T16:37:58,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078289 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078296 2024-11-11T16:37:58,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,303 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=771, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,303 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078296) roll requested 2024-11-11T16:37:58,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078303 2024-11-11T16:37:58,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078296 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078303 2024-11-11T16:37:58,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,310 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=772, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,310 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078303) roll requested 2024-11-11T16:37:58,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078310 2024-11-11T16:37:58,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078310 2024-11-11T16:37:58,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,317 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=773, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078310) roll requested 2024-11-11T16:37:58,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078317 2024-11-11T16:37:58,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078310 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078317 2024-11-11T16:37:58,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,323 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,323 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=774, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,324 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078317) roll requested 2024-11-11T16:37:58,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078324 2024-11-11T16:37:58,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078324 2024-11-11T16:37:58,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,330 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,331 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=775, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,331 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078324) roll requested 2024-11-11T16:37:58,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078331 2024-11-11T16:37:58,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078324 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078331 2024-11-11T16:37:58,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,337 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,337 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=776, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078331) roll requested 2024-11-11T16:37:58,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078337 2024-11-11T16:37:58,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078331 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078337 2024-11-11T16:37:58,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=777, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,344 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078337) roll requested 2024-11-11T16:37:58,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078344 2024-11-11T16:37:58,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078344 2024-11-11T16:37:58,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,351 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,351 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=778, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,351 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078344) roll requested 2024-11-11T16:37:58,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078351 2024-11-11T16:37:58,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078344 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078351 2024-11-11T16:37:58,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,359 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,359 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=779, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,359 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078351) roll requested 2024-11-11T16:37:58,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078359 2024-11-11T16:37:58,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078351 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078359 2024-11-11T16:37:58,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,366 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078351 is not closed yet, will try archiving it next time 2024-11-11T16:37:58,366 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=780, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,366 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078359) roll requested 2024-11-11T16:37:58,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078366 2024-11-11T16:37:58,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078359 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078366 2024-11-11T16:37:58,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,376 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,376 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=781, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,376 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078366) roll requested 2024-11-11T16:37:58,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078376 2024-11-11T16:37:58,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078366 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078376 2024-11-11T16:37:58,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=782, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,387 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078376) roll requested 2024-11-11T16:37:58,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078387 2024-11-11T16:37:58,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078376 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078387 2024-11-11T16:37:58,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,395 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,395 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=783, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,395 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078387) roll requested 2024-11-11T16:37:58,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078395 2024-11-11T16:37:58,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078395 2024-11-11T16:37:58,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,406 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=784, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,406 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,406 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078395) roll requested 2024-11-11T16:37:58,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078406 2024-11-11T16:37:58,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078395 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078406 2024-11-11T16:37:58,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=785, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078406) roll requested 2024-11-11T16:37:58,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078416 2024-11-11T16:37:58,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078406 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078416 2024-11-11T16:37:58,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,433 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=786, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,433 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078416) roll requested 2024-11-11T16:37:58,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078433 2024-11-11T16:37:58,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078416 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078433 2024-11-11T16:37:58,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,441 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=787, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078433) roll requested 2024-11-11T16:37:58,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078442 2024-11-11T16:37:58,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078442 2024-11-11T16:37:58,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,451 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=788, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,451 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078442) roll requested 2024-11-11T16:37:58,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078451 2024-11-11T16:37:58,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078451 2024-11-11T16:37:58,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,460 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=789, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,460 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078451) roll requested 2024-11-11T16:37:58,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078461 2024-11-11T16:37:58,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078451 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078461 2024-11-11T16:37:58,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=790, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078461) roll requested 2024-11-11T16:37:58,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078475 2024-11-11T16:37:58,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078475 2024-11-11T16:37:58,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,490 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=791, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,490 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078475) roll requested 2024-11-11T16:37:58,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078490 2024-11-11T16:37:58,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078490 2024-11-11T16:37:58,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,501 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=792, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078490) roll requested 2024-11-11T16:37:58,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078501 2024-11-11T16:37:58,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078490 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078501 2024-11-11T16:37:58,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=793, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078501) roll requested 2024-11-11T16:37:58,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078512 2024-11-11T16:37:58,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078512 2024-11-11T16:37:58,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=794, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078512) roll requested 2024-11-11T16:37:58,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078524 2024-11-11T16:37:58,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078512 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078524 2024-11-11T16:37:58,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,530 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=795, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,531 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078524) roll requested 2024-11-11T16:37:58,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078531 2024-11-11T16:37:58,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078524 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078531 2024-11-11T16:37:58,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,538 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,538 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=796, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,538 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078531) roll requested 2024-11-11T16:37:58,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078538 2024-11-11T16:37:58,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078538 2024-11-11T16:37:58,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=797, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078538) roll requested 2024-11-11T16:37:58,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078550 2024-11-11T16:37:58,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078538 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078550 2024-11-11T16:37:58,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,569 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,569 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=798, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078550) roll requested 2024-11-11T16:37:58,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078569 2024-11-11T16:37:58,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078569 2024-11-11T16:37:58,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,581 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,581 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=799, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,581 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078569) roll requested 2024-11-11T16:37:58,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078581 2024-11-11T16:37:58,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078581 2024-11-11T16:37:58,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,588 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=800, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,588 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078581) roll requested 2024-11-11T16:37:58,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078588 2024-11-11T16:37:58,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078588 2024-11-11T16:37:58,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,599 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=801, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,599 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078588) roll requested 2024-11-11T16:37:58,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078599 2024-11-11T16:37:58,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078588 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078599 2024-11-11T16:37:58,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,605 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,605 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=802, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,605 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078599) roll requested 2024-11-11T16:37:58,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078606 2024-11-11T16:37:58,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078606 2024-11-11T16:37:58,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,612 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=803, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,612 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078606) roll requested 2024-11-11T16:37:58,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078612 2024-11-11T16:37:58,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078606 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078612 2024-11-11T16:37:58,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,618 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=804, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078612) roll requested 2024-11-11T16:37:58,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078619 2024-11-11T16:37:58,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078612 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078619 2024-11-11T16:37:58,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,625 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,625 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=805, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078619) roll requested 2024-11-11T16:37:58,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078625 2024-11-11T16:37:58,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078619 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078625 2024-11-11T16:37:58,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,631 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,631 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=806, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,632 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078625) roll requested 2024-11-11T16:37:58,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078632 2024-11-11T16:37:58,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078632 2024-11-11T16:37:58,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,638 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,638 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=807, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,638 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078632) roll requested 2024-11-11T16:37:58,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078638 2024-11-11T16:37:58,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078632 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078638 2024-11-11T16:37:58,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,644 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=808, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078638) roll requested 2024-11-11T16:37:58,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078645 2024-11-11T16:37:58,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078645 2024-11-11T16:37:58,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,653 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=809, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,653 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078645) roll requested 2024-11-11T16:37:58,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078653 2024-11-11T16:37:58,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078653 2024-11-11T16:37:58,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,660 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,660 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=810, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,660 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078653) roll requested 2024-11-11T16:37:58,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078660 2024-11-11T16:37:58,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078660 2024-11-11T16:37:58,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=811, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,669 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078660) roll requested 2024-11-11T16:37:58,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078669 2024-11-11T16:37:58,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078669 2024-11-11T16:37:58,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,680 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=812, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,680 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078669) roll requested 2024-11-11T16:37:58,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078680 2024-11-11T16:37:58,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078669 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078680 2024-11-11T16:37:58,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,690 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,691 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=813, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,691 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078680) roll requested 2024-11-11T16:37:58,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078691 2024-11-11T16:37:58,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078680 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078691 2024-11-11T16:37:58,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,704 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,705 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=814, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,705 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078691) roll requested 2024-11-11T16:37:58,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078705 2024-11-11T16:37:58,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078705 2024-11-11T16:37:58,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=815, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078705) roll requested 2024-11-11T16:37:58,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078717 2024-11-11T16:37:58,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078705 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078717 2024-11-11T16:37:58,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,727 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,727 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=816, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078717) roll requested 2024-11-11T16:37:58,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078728 2024-11-11T16:37:58,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078728 2024-11-11T16:37:58,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,745 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,745 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=817, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,745 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078728) roll requested 2024-11-11T16:37:58,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078745 2024-11-11T16:37:58,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078745 2024-11-11T16:37:58,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,757 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=818, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,757 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078745) roll requested 2024-11-11T16:37:58,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078757 2024-11-11T16:37:58,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078757 2024-11-11T16:37:58,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=819, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078757) roll requested 2024-11-11T16:37:58,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078777 2024-11-11T16:37:58,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078777 2024-11-11T16:37:58,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,786 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=820, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,786 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078777) roll requested 2024-11-11T16:37:58,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078786 2024-11-11T16:37:58,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078786 2024-11-11T16:37:58,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=821, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,793 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078786) roll requested 2024-11-11T16:37:58,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078793 2024-11-11T16:37:58,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078786 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078793 2024-11-11T16:37:58,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=822, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078793) roll requested 2024-11-11T16:37:58,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078801 2024-11-11T16:37:58,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078801 2024-11-11T16:37:58,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,807 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=823, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078801) roll requested 2024-11-11T16:37:58,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078808 2024-11-11T16:37:58,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078808 2024-11-11T16:37:58,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=824, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,816 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078808) roll requested 2024-11-11T16:37:58,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078816 2024-11-11T16:37:58,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078816 2024-11-11T16:37:58,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=825, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078816) roll requested 2024-11-11T16:37:58,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078823 2024-11-11T16:37:58,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078816 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078823 2024-11-11T16:37:58,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=826, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,832 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078823) roll requested 2024-11-11T16:37:58,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078833 2024-11-11T16:37:58,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078833 2024-11-11T16:37:58,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,839 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,840 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=827, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,840 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078833) roll requested 2024-11-11T16:37:58,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078840 2024-11-11T16:37:58,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078833 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078840 2024-11-11T16:37:58,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=828, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078840) roll requested 2024-11-11T16:37:58,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078847 2024-11-11T16:37:58,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078847 2024-11-11T16:37:58,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,855 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=829, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078847) roll requested 2024-11-11T16:37:58,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078856 2024-11-11T16:37:58,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078856 2024-11-11T16:37:58,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,863 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=830, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078856) roll requested 2024-11-11T16:37:58,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078864 2024-11-11T16:37:58,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078864 2024-11-11T16:37:58,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=831, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078864) roll requested 2024-11-11T16:37:58,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078874 2024-11-11T16:37:58,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078864 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078874 2024-11-11T16:37:58,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=832, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078874) roll requested 2024-11-11T16:37:58,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078881 2024-11-11T16:37:58,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078881 2024-11-11T16:37:58,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,889 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,889 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=833, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,889 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078881) roll requested 2024-11-11T16:37:58,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078890 2024-11-11T16:37:58,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078890 2024-11-11T16:37:58,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,898 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=834, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,898 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078890) roll requested 2024-11-11T16:37:58,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078898 2024-11-11T16:37:58,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078898 2024-11-11T16:37:58,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=835, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078898) roll requested 2024-11-11T16:37:58,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078906 2024-11-11T16:37:58,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078898 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078906 2024-11-11T16:37:58,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,914 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,914 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=836, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,914 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078906) roll requested 2024-11-11T16:37:58,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078914 2024-11-11T16:37:58,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078914 2024-11-11T16:37:58,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,922 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,922 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=837, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,922 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078914) roll requested 2024-11-11T16:37:58,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078923 2024-11-11T16:37:58,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078914 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078923 2024-11-11T16:37:58,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,930 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,930 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=838, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078923) roll requested 2024-11-11T16:37:58,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078930 2024-11-11T16:37:58,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078923 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078930 2024-11-11T16:37:58,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,937 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=839, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,937 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078930) roll requested 2024-11-11T16:37:58,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078937 2024-11-11T16:37:58,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078930 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078937 2024-11-11T16:37:58,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=840, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078937) roll requested 2024-11-11T16:37:58,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078944 2024-11-11T16:37:58,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078944 2024-11-11T16:37:58,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,950 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,951 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=841, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,951 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078944) roll requested 2024-11-11T16:37:58,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078951 2024-11-11T16:37:58,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078951 2024-11-11T16:37:58,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,957 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,958 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=842, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,958 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078951) roll requested 2024-11-11T16:37:58,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078958 2024-11-11T16:37:58,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078951 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078958 2024-11-11T16:37:58,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=843, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078958) roll requested 2024-11-11T16:37:58,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078965 2024-11-11T16:37:58,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078958 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078965 2024-11-11T16:37:58,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=844, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078965) roll requested 2024-11-11T16:37:58,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078972 2024-11-11T16:37:58,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078972 2024-11-11T16:37:58,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=845, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078972) roll requested 2024-11-11T16:37:58,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078979 2024-11-11T16:37:58,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078979 2024-11-11T16:37:58,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,985 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=846, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,985 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078979) roll requested 2024-11-11T16:37:58,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078985 2024-11-11T16:37:58,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078985 2024-11-11T16:37:58,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,992 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=847, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,992 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078985) roll requested 2024-11-11T16:37:58,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078992 2024-11-11T16:37:58,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:58,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078992 2024-11-11T16:37:58,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:58,998 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:58,998 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=848, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:58,998 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:58,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078992) roll requested 2024-11-11T16:37:58,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343078998 2024-11-11T16:37:59,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078992 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078998 2024-11-11T16:37:59,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,005 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=849, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343078998) roll requested 2024-11-11T16:37:59,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079005 2024-11-11T16:37:59,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343078998 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079005 2024-11-11T16:37:59,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=850, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079005) roll requested 2024-11-11T16:37:59,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079012 2024-11-11T16:37:59,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079012 2024-11-11T16:37:59,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=851, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079012) roll requested 2024-11-11T16:37:59,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079018 2024-11-11T16:37:59,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079018 2024-11-11T16:37:59,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=852, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079018) roll requested 2024-11-11T16:37:59,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079025 2024-11-11T16:37:59,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079025 2024-11-11T16:37:59,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,031 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,031 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=853, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,031 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079025) roll requested 2024-11-11T16:37:59,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079031 2024-11-11T16:37:59,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079031 2024-11-11T16:37:59,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,038 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,038 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=854, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,038 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079031) roll requested 2024-11-11T16:37:59,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079038 2024-11-11T16:37:59,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079031 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079038 2024-11-11T16:37:59,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,045 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=855, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079038) roll requested 2024-11-11T16:37:59,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079045 2024-11-11T16:37:59,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079045 2024-11-11T16:37:59,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,051 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=856, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,052 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079045) roll requested 2024-11-11T16:37:59,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079052 2024-11-11T16:37:59,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079052 2024-11-11T16:37:59,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,058 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,058 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=857, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,058 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079052) roll requested 2024-11-11T16:37:59,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079058 2024-11-11T16:37:59,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079052 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079058 2024-11-11T16:37:59,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,065 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=858, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,065 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079058) roll requested 2024-11-11T16:37:59,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079065 2024-11-11T16:37:59,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079058 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079065 2024-11-11T16:37:59,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,071 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,071 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=859, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,071 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079065) roll requested 2024-11-11T16:37:59,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079071 2024-11-11T16:37:59,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079065 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079071 2024-11-11T16:37:59,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=860, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079071) roll requested 2024-11-11T16:37:59,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079079 2024-11-11T16:37:59,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079071 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079079 2024-11-11T16:37:59,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=861, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,087 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079079) roll requested 2024-11-11T16:37:59,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079087 2024-11-11T16:37:59,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079087 2024-11-11T16:37:59,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,093 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,094 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=862, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,094 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079087) roll requested 2024-11-11T16:37:59,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079094 2024-11-11T16:37:59,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079087 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079094 2024-11-11T16:37:59,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,100 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=863, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,101 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079094) roll requested 2024-11-11T16:37:59,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079101 2024-11-11T16:37:59,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079094 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079101 2024-11-11T16:37:59,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,107 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,107 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=864, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,107 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079101) roll requested 2024-11-11T16:37:59,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079108 2024-11-11T16:37:59,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079108 2024-11-11T16:37:59,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,114 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,114 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=865, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,114 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079108) roll requested 2024-11-11T16:37:59,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079115 2024-11-11T16:37:59,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079108 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079115 2024-11-11T16:37:59,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,121 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=866, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079115) roll requested 2024-11-11T16:37:59,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079121 2024-11-11T16:37:59,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079115 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079121 2024-11-11T16:37:59,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,128 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=867, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,128 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079121) roll requested 2024-11-11T16:37:59,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079128 2024-11-11T16:37:59,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079128 2024-11-11T16:37:59,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,135 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=868, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,135 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079128) roll requested 2024-11-11T16:37:59,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079135 2024-11-11T16:37:59,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079128 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079135 2024-11-11T16:37:59,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,142 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,142 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=869, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,142 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079135) roll requested 2024-11-11T16:37:59,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079142 2024-11-11T16:37:59,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079135 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079142 2024-11-11T16:37:59,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=870, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079142) roll requested 2024-11-11T16:37:59,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079149 2024-11-11T16:37:59,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079142 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079149 2024-11-11T16:37:59,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=871, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079149) roll requested 2024-11-11T16:37:59,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079155 2024-11-11T16:37:59,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079155 2024-11-11T16:37:59,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=872, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079155) roll requested 2024-11-11T16:37:59,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079162 2024-11-11T16:37:59,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079162 2024-11-11T16:37:59,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=873, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,169 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079162) roll requested 2024-11-11T16:37:59,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079169 2024-11-11T16:37:59,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079169 2024-11-11T16:37:59,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,175 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,176 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=874, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,176 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079169) roll requested 2024-11-11T16:37:59,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079176 2024-11-11T16:37:59,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079169 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079176 2024-11-11T16:37:59,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,186 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=875, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,186 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079176) roll requested 2024-11-11T16:37:59,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079186 2024-11-11T16:37:59,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079176 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079186 2024-11-11T16:37:59,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=876, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079186) roll requested 2024-11-11T16:37:59,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079196 2024-11-11T16:37:59,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079186 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079196 2024-11-11T16:37:59,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=877, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079196) roll requested 2024-11-11T16:37:59,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079203 2024-11-11T16:37:59,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079203 2024-11-11T16:37:59,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=878, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079203) roll requested 2024-11-11T16:37:59,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079211 2024-11-11T16:37:59,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079211 2024-11-11T16:37:59,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,220 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,220 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=879, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,220 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079211) roll requested 2024-11-11T16:37:59,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079220 2024-11-11T16:37:59,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079220 2024-11-11T16:37:59,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,227 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,227 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=880, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,227 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079220) roll requested 2024-11-11T16:37:59,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079227 2024-11-11T16:37:59,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079220 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079227 2024-11-11T16:37:59,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,235 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,236 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=881, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,236 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079227) roll requested 2024-11-11T16:37:59,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079236 2024-11-11T16:37:59,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079227 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079236 2024-11-11T16:37:59,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,243 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,243 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=882, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,243 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079236) roll requested 2024-11-11T16:37:59,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079243 2024-11-11T16:37:59,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079236 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079243 2024-11-11T16:37:59,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,251 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,251 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=883, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,251 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079243) roll requested 2024-11-11T16:37:59,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079251 2024-11-11T16:37:59,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079243 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079251 2024-11-11T16:37:59,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,258 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=884, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,258 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079251) roll requested 2024-11-11T16:37:59,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079258 2024-11-11T16:37:59,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079251 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079258 2024-11-11T16:37:59,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,265 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=885, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,265 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079258) roll requested 2024-11-11T16:37:59,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079265 2024-11-11T16:37:59,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079258 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079265 2024-11-11T16:37:59,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=886, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,273 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079265) roll requested 2024-11-11T16:37:59,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079273 2024-11-11T16:37:59,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079265 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079273 2024-11-11T16:37:59,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,279 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=887, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,279 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079273) roll requested 2024-11-11T16:37:59,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079279 2024-11-11T16:37:59,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079279 2024-11-11T16:37:59,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,286 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,286 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=888, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,286 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079279) roll requested 2024-11-11T16:37:59,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079286 2024-11-11T16:37:59,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079279 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079286 2024-11-11T16:37:59,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=889, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,293 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079286) roll requested 2024-11-11T16:37:59,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079293 2024-11-11T16:37:59,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079286 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079293 2024-11-11T16:37:59,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,300 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=890, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,300 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079293) roll requested 2024-11-11T16:37:59,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079300 2024-11-11T16:37:59,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079300 2024-11-11T16:37:59,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,306 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,306 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=891, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,307 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079300) roll requested 2024-11-11T16:37:59,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079307 2024-11-11T16:37:59,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079300 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079307 2024-11-11T16:37:59,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,314 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,315 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=892, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,315 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079307) roll requested 2024-11-11T16:37:59,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079315 2024-11-11T16:37:59,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079307 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079315 2024-11-11T16:37:59,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,321 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,321 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=893, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079315) roll requested 2024-11-11T16:37:59,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079321 2024-11-11T16:37:59,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079315 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079321 2024-11-11T16:37:59,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=894, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079321) roll requested 2024-11-11T16:37:59,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079329 2024-11-11T16:37:59,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079329 2024-11-11T16:37:59,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,340 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=895, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,340 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079329) roll requested 2024-11-11T16:37:59,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079340 2024-11-11T16:37:59,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079340 2024-11-11T16:37:59,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=896, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079340) roll requested 2024-11-11T16:37:59,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079349 2024-11-11T16:37:59,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079340 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079349 2024-11-11T16:37:59,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=897, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079349) roll requested 2024-11-11T16:37:59,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079355 2024-11-11T16:37:59,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079355 2024-11-11T16:37:59,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,362 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=898, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,362 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079355) roll requested 2024-11-11T16:37:59,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079362 2024-11-11T16:37:59,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079362 2024-11-11T16:37:59,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=899, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,369 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079362) roll requested 2024-11-11T16:37:59,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079369 2024-11-11T16:37:59,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079362 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079369 2024-11-11T16:37:59,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,376 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,376 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=900, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,376 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079369) roll requested 2024-11-11T16:37:59,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079376 2024-11-11T16:37:59,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079376 2024-11-11T16:37:59,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,383 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,384 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=901, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,384 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079376) roll requested 2024-11-11T16:37:59,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079384 2024-11-11T16:37:59,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079376 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079384 2024-11-11T16:37:59,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=902, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,390 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079384) roll requested 2024-11-11T16:37:59,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079390 2024-11-11T16:37:59,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079384 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079390 2024-11-11T16:37:59,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=903, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079390) roll requested 2024-11-11T16:37:59,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079397 2024-11-11T16:37:59,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079397 2024-11-11T16:37:59,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,404 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,404 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=904, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,404 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079397) roll requested 2024-11-11T16:37:59,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079404 2024-11-11T16:37:59,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079404 2024-11-11T16:37:59,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,410 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,410 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=905, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,410 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079404) roll requested 2024-11-11T16:37:59,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079410 2024-11-11T16:37:59,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079410 2024-11-11T16:37:59,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=906, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079410) roll requested 2024-11-11T16:37:59,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079417 2024-11-11T16:37:59,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079410 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079417 2024-11-11T16:37:59,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=907, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079417) roll requested 2024-11-11T16:37:59,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079424 2024-11-11T16:37:59,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079424 2024-11-11T16:37:59,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,430 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,430 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=908, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,430 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079424) roll requested 2024-11-11T16:37:59,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079430 2024-11-11T16:37:59,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079430 2024-11-11T16:37:59,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,437 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=909, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,437 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079430) roll requested 2024-11-11T16:37:59,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079437 2024-11-11T16:37:59,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079430 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079437 2024-11-11T16:37:59,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,443 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=910, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,443 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079437) roll requested 2024-11-11T16:37:59,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079443 2024-11-11T16:37:59,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079437 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079443 2024-11-11T16:37:59,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,451 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=911, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,451 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079443) roll requested 2024-11-11T16:37:59,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079451 2024-11-11T16:37:59,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079443 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079451 2024-11-11T16:37:59,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,461 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=912, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079451) roll requested 2024-11-11T16:37:59,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079461 2024-11-11T16:37:59,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079451 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079461 2024-11-11T16:37:59,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=913, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,468 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079461) roll requested 2024-11-11T16:37:59,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079468 2024-11-11T16:37:59,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079468 2024-11-11T16:37:59,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=914, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079468) roll requested 2024-11-11T16:37:59,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079475 2024-11-11T16:37:59,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079468 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079475 2024-11-11T16:37:59,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=915, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079475) roll requested 2024-11-11T16:37:59,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079485 2024-11-11T16:37:59,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079485 2024-11-11T16:37:59,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=916, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079485) roll requested 2024-11-11T16:37:59,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079493 2024-11-11T16:37:59,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079493 2024-11-11T16:37:59,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=917, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079493) roll requested 2024-11-11T16:37:59,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079503 2024-11-11T16:37:59,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079503 2024-11-11T16:37:59,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=918, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079503) roll requested 2024-11-11T16:37:59,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079511 2024-11-11T16:37:59,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079511 2024-11-11T16:37:59,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,519 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,519 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=919, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079511) roll requested 2024-11-11T16:37:59,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079520 2024-11-11T16:37:59,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079520 2024-11-11T16:37:59,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,527 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,527 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=920, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079520) roll requested 2024-11-11T16:37:59,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079527 2024-11-11T16:37:59,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079520 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079527 2024-11-11T16:37:59,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,534 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=921, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,534 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079527) roll requested 2024-11-11T16:37:59,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079534 2024-11-11T16:37:59,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079534 2024-11-11T16:37:59,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,541 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,541 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=922, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,541 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079534) roll requested 2024-11-11T16:37:59,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079541 2024-11-11T16:37:59,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079534 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079541 2024-11-11T16:37:59,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,548 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=923, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,549 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079541) roll requested 2024-11-11T16:37:59,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079549 2024-11-11T16:37:59,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079549 2024-11-11T16:37:59,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,560 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,560 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=924, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,560 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079549) roll requested 2024-11-11T16:37:59,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079560 2024-11-11T16:37:59,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079549 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079560 2024-11-11T16:37:59,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,569 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=925, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079560) roll requested 2024-11-11T16:37:59,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079569 2024-11-11T16:37:59,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079560 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079569 2024-11-11T16:37:59,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,576 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,576 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=926, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,576 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079569) roll requested 2024-11-11T16:37:59,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079576 2024-11-11T16:37:59,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079576 2024-11-11T16:37:59,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,583 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,583 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=927, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,583 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079576) roll requested 2024-11-11T16:37:59,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079583 2024-11-11T16:37:59,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079576 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079583 2024-11-11T16:37:59,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,590 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,590 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=928, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,590 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079583) roll requested 2024-11-11T16:37:59,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079591 2024-11-11T16:37:59,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079583 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079591 2024-11-11T16:37:59,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,597 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,597 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=929, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,597 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079591) roll requested 2024-11-11T16:37:59,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079597 2024-11-11T16:37:59,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079591 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079597 2024-11-11T16:37:59,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,603 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,603 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=930, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,603 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079597) roll requested 2024-11-11T16:37:59,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079604 2024-11-11T16:37:59,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079597 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079604 2024-11-11T16:37:59,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=931, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079604) roll requested 2024-11-11T16:37:59,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079610 2024-11-11T16:37:59,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079610 2024-11-11T16:37:59,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,617 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=932, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079610) roll requested 2024-11-11T16:37:59,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079618 2024-11-11T16:37:59,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079618 2024-11-11T16:37:59,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=933, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079618) roll requested 2024-11-11T16:37:59,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079624 2024-11-11T16:37:59,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079618 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079624 2024-11-11T16:37:59,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,631 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,631 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=934, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,631 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079624) roll requested 2024-11-11T16:37:59,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079631 2024-11-11T16:37:59,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079631 2024-11-11T16:37:59,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,638 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=935, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,638 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079631) roll requested 2024-11-11T16:37:59,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079638 2024-11-11T16:37:59,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079631 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079638 2024-11-11T16:37:59,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=936, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079638) roll requested 2024-11-11T16:37:59,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079645 2024-11-11T16:37:59,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079645 2024-11-11T16:37:59,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=937, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079645) roll requested 2024-11-11T16:37:59,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079652 2024-11-11T16:37:59,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079652 2024-11-11T16:37:59,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,661 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,661 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=938, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,661 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079652) roll requested 2024-11-11T16:37:59,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079661 2024-11-11T16:37:59,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079661 2024-11-11T16:37:59,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,667 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,668 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=939, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,668 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079661) roll requested 2024-11-11T16:37:59,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079668 2024-11-11T16:37:59,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079661 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079668 2024-11-11T16:37:59,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=940, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079668) roll requested 2024-11-11T16:37:59,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079676 2024-11-11T16:37:59,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079668 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079676 2024-11-11T16:37:59,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,683 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,683 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=941, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,683 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079676) roll requested 2024-11-11T16:37:59,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079683 2024-11-11T16:37:59,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079676 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079683 2024-11-11T16:37:59,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,693 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=942, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,693 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079683) roll requested 2024-11-11T16:37:59,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079693 2024-11-11T16:37:59,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079683 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079693 2024-11-11T16:37:59,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,700 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=943, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,700 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079693) roll requested 2024-11-11T16:37:59,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079700 2024-11-11T16:37:59,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079693 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079700 2024-11-11T16:37:59,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,706 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,707 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=944, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,707 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079700) roll requested 2024-11-11T16:37:59,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079707 2024-11-11T16:37:59,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079700 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079707 2024-11-11T16:37:59,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,713 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=945, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079707) roll requested 2024-11-11T16:37:59,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079714 2024-11-11T16:37:59,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079707 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079714 2024-11-11T16:37:59,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,720 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,721 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=946, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,721 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079714) roll requested 2024-11-11T16:37:59,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079721 2024-11-11T16:37:59,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079721 2024-11-11T16:37:59,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,727 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,727 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=947, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,727 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079721) roll requested 2024-11-11T16:37:59,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079727 2024-11-11T16:37:59,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079727 2024-11-11T16:37:59,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,736 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,737 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=948, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,737 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079727) roll requested 2024-11-11T16:37:59,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079737 2024-11-11T16:37:59,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079727 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079737 2024-11-11T16:37:59,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,743 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,743 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=949, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,743 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079737) roll requested 2024-11-11T16:37:59,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079743 2024-11-11T16:37:59,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079743 2024-11-11T16:37:59,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,751 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,751 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=950, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,751 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079743) roll requested 2024-11-11T16:37:59,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079751 2024-11-11T16:37:59,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079743 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079751 2024-11-11T16:37:59,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,758 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,758 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=951, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,758 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079751) roll requested 2024-11-11T16:37:59,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079758 2024-11-11T16:37:59,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079751 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079758 2024-11-11T16:37:59,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=952, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079758) roll requested 2024-11-11T16:37:59,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079767 2024-11-11T16:37:59,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079758 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079767 2024-11-11T16:37:59,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=953, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079767) roll requested 2024-11-11T16:37:59,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079773 2024-11-11T16:37:59,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079773 2024-11-11T16:37:59,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,780 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,780 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=954, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,780 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079773) roll requested 2024-11-11T16:37:59,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079780 2024-11-11T16:37:59,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079780 2024-11-11T16:37:59,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,786 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,786 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=955, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,787 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079780) roll requested 2024-11-11T16:37:59,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079787 2024-11-11T16:37:59,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079780 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079787 2024-11-11T16:37:59,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=956, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,793 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079787) roll requested 2024-11-11T16:37:59,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079793 2024-11-11T16:37:59,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079787 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079793 2024-11-11T16:37:59,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=957, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079793) roll requested 2024-11-11T16:37:59,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079801 2024-11-11T16:37:59,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079801 2024-11-11T16:37:59,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,807 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,807 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=958, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,807 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079801) roll requested 2024-11-11T16:37:59,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079807 2024-11-11T16:37:59,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079807 2024-11-11T16:37:59,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,814 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=959, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,814 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079807) roll requested 2024-11-11T16:37:59,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079814 2024-11-11T16:37:59,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079807 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079814 2024-11-11T16:37:59,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=960, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079814) roll requested 2024-11-11T16:37:59,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079821 2024-11-11T16:37:59,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079814 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079821 2024-11-11T16:37:59,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=961, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079821) roll requested 2024-11-11T16:37:59,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079828 2024-11-11T16:37:59,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079828 2024-11-11T16:37:59,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,835 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=962, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079828) roll requested 2024-11-11T16:37:59,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079835 2024-11-11T16:37:59,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079835 2024-11-11T16:37:59,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,841 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,841 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=963, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,841 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079835) roll requested 2024-11-11T16:37:59,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079842 2024-11-11T16:37:59,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079842 2024-11-11T16:37:59,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=964, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079842) roll requested 2024-11-11T16:37:59,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079849 2024-11-11T16:37:59,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079849 2024-11-11T16:37:59,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=965, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079849) roll requested 2024-11-11T16:37:59,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079856 2024-11-11T16:37:59,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079856 2024-11-11T16:37:59,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=966, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,866 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079856) roll requested 2024-11-11T16:37:59,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079866 2024-11-11T16:37:59,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079866 2024-11-11T16:37:59,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=967, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079866) roll requested 2024-11-11T16:37:59,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079873 2024-11-11T16:37:59,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079866 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079873 2024-11-11T16:37:59,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=968, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079873) roll requested 2024-11-11T16:37:59,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079881 2024-11-11T16:37:59,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079881 2024-11-11T16:37:59,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,891 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,891 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=969, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,891 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079881) roll requested 2024-11-11T16:37:59,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079891 2024-11-11T16:37:59,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079891 2024-11-11T16:37:59,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,903 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,904 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=970, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,904 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079891) roll requested 2024-11-11T16:37:59,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079904 2024-11-11T16:37:59,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079891 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079904 2024-11-11T16:37:59,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,911 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=971, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,911 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079904) roll requested 2024-11-11T16:37:59,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079911 2024-11-11T16:37:59,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079904 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079911 2024-11-11T16:37:59,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=972, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079911) roll requested 2024-11-11T16:37:59,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079922 2024-11-11T16:37:59,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079911 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079922 2024-11-11T16:37:59,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,937 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=973, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,937 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079922) roll requested 2024-11-11T16:37:59,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079937 2024-11-11T16:37:59,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079922 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079937 2024-11-11T16:37:59,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,946 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,947 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=974, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,947 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079937) roll requested 2024-11-11T16:37:59,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079947 2024-11-11T16:37:59,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079947 2024-11-11T16:37:59,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,957 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=975, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,957 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079947) roll requested 2024-11-11T16:37:59,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079957 2024-11-11T16:37:59,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079947 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079957 2024-11-11T16:37:59,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=976, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079957) roll requested 2024-11-11T16:37:59,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079964 2024-11-11T16:37:59,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079957 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079964 2024-11-11T16:37:59,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=977, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079964) roll requested 2024-11-11T16:37:59,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079971 2024-11-11T16:37:59,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079971 2024-11-11T16:37:59,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,978 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=978, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,978 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079971) roll requested 2024-11-11T16:37:59,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079978 2024-11-11T16:37:59,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079978 2024-11-11T16:37:59,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,985 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=979, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,985 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079978) roll requested 2024-11-11T16:37:59,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079985 2024-11-11T16:37:59,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079985 2024-11-11T16:37:59,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:37:59,992 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:37:59,992 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=980, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:37:59,992 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:37:59,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079985) roll requested 2024-11-11T16:37:59,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343079992 2024-11-11T16:37:59,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:37:59,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079992 2024-11-11T16:38:00,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=981, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,001 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343079992) roll requested 2024-11-11T16:38:00,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080001 2024-11-11T16:38:00,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343079992 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080001 2024-11-11T16:38:00,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,007 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=982, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,008 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080001) roll requested 2024-11-11T16:38:00,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080008 2024-11-11T16:38:00,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080001 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080008 2024-11-11T16:38:00,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=983, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080008) roll requested 2024-11-11T16:38:00,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080014 2024-11-11T16:38:00,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080008 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080014 2024-11-11T16:38:00,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=984, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080014) roll requested 2024-11-11T16:38:00,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080021 2024-11-11T16:38:00,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080021 2024-11-11T16:38:00,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,027 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,027 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=985, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080021) roll requested 2024-11-11T16:38:00,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080028 2024-11-11T16:38:00,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080028 2024-11-11T16:38:00,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=986, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080028) roll requested 2024-11-11T16:38:00,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080034 2024-11-11T16:38:00,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080034 2024-11-11T16:38:00,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=987, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080034) roll requested 2024-11-11T16:38:00,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080041 2024-11-11T16:38:00,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080034 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080041 2024-11-11T16:38:00,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=988, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080041) roll requested 2024-11-11T16:38:00,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080048 2024-11-11T16:38:00,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080048 2024-11-11T16:38:00,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=989, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080048) roll requested 2024-11-11T16:38:00,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080055 2024-11-11T16:38:00,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080055 2024-11-11T16:38:00,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,061 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=990, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080055) roll requested 2024-11-11T16:38:00,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080061 2024-11-11T16:38:00,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080061 2024-11-11T16:38:00,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,067 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=991, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,067 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080061) roll requested 2024-11-11T16:38:00,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080067 2024-11-11T16:38:00,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080067 2024-11-11T16:38:00,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,074 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,074 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=992, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,074 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080067) roll requested 2024-11-11T16:38:00,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080074 2024-11-11T16:38:00,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080067 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080074 2024-11-11T16:38:00,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,081 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,081 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=993, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,081 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080074) roll requested 2024-11-11T16:38:00,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080081 2024-11-11T16:38:00,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080074 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080081 2024-11-11T16:38:00,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,087 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,087 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=994, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,087 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080081) roll requested 2024-11-11T16:38:00,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080087 2024-11-11T16:38:00,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080081 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080087 2024-11-11T16:38:00,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,093 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,094 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=995, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,094 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080087) roll requested 2024-11-11T16:38:00,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080094 2024-11-11T16:38:00,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080087 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080094 2024-11-11T16:38:00,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,100 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,100 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=996, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,100 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080094) roll requested 2024-11-11T16:38:00,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080100 2024-11-11T16:38:00,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080094 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080100 2024-11-11T16:38:00,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,106 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,107 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=997, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,107 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080100) roll requested 2024-11-11T16:38:00,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080107 2024-11-11T16:38:00,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080100 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080107 2024-11-11T16:38:00,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,113 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,113 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=998, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080107) roll requested 2024-11-11T16:38:00,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080113 2024-11-11T16:38:00,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080107 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080113 2024-11-11T16:38:00,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,119 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=999, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,120 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080113) roll requested 2024-11-11T16:38:00,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080120 2024-11-11T16:38:00,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080120 2024-11-11T16:38:00,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,126 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1000, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080120) roll requested 2024-11-11T16:38:00,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080126 2024-11-11T16:38:00,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080120 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080126 2024-11-11T16:38:00,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,132 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,132 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1001, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,132 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080126) roll requested 2024-11-11T16:38:00,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080132 2024-11-11T16:38:00,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080132 2024-11-11T16:38:00,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,139 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,139 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1002, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,139 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080132) roll requested 2024-11-11T16:38:00,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080139 2024-11-11T16:38:00,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080132 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080139 2024-11-11T16:38:00,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,145 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,146 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1003, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,146 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080139) roll requested 2024-11-11T16:38:00,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080146 2024-11-11T16:38:00,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080146 2024-11-11T16:38:00,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,152 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1004, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,152 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080146) roll requested 2024-11-11T16:38:00,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080152 2024-11-11T16:38:00,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080146 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080152 2024-11-11T16:38:00,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,158 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1005, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080152) roll requested 2024-11-11T16:38:00,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080159 2024-11-11T16:38:00,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080152 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080159 2024-11-11T16:38:00,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1006, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080159) roll requested 2024-11-11T16:38:00,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080166 2024-11-11T16:38:00,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080159 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080166 2024-11-11T16:38:00,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1007, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080166) roll requested 2024-11-11T16:38:00,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080173 2024-11-11T16:38:00,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080173 2024-11-11T16:38:00,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,181 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1008, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,181 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080173) roll requested 2024-11-11T16:38:00,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080181 2024-11-11T16:38:00,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080181 2024-11-11T16:38:00,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1009, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080181) roll requested 2024-11-11T16:38:00,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080188 2024-11-11T16:38:00,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080181 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080188 2024-11-11T16:38:00,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1010, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080188) roll requested 2024-11-11T16:38:00,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080195 2024-11-11T16:38:00,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080188 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080195 2024-11-11T16:38:00,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,204 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1011, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,204 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080195) roll requested 2024-11-11T16:38:00,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080204 2024-11-11T16:38:00,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080204 2024-11-11T16:38:00,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1012, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080204) roll requested 2024-11-11T16:38:00,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080211 2024-11-11T16:38:00,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080211 2024-11-11T16:38:00,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1013, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080211) roll requested 2024-11-11T16:38:00,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080218 2024-11-11T16:38:00,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080218 2024-11-11T16:38:00,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1014, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080218) roll requested 2024-11-11T16:38:00,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080225 2024-11-11T16:38:00,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080225 2024-11-11T16:38:00,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,231 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1015, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080225) roll requested 2024-11-11T16:38:00,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080232 2024-11-11T16:38:00,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080232 2024-11-11T16:38:00,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,238 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,238 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1016, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,238 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080232) roll requested 2024-11-11T16:38:00,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080238 2024-11-11T16:38:00,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080232 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080238 2024-11-11T16:38:00,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1017, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080238) roll requested 2024-11-11T16:38:00,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080245 2024-11-11T16:38:00,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080238 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080245 2024-11-11T16:38:00,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,251 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1018, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080245) roll requested 2024-11-11T16:38:00,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080252 2024-11-11T16:38:00,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080252 2024-11-11T16:38:00,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,258 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1019, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,258 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080252) roll requested 2024-11-11T16:38:00,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080258 2024-11-11T16:38:00,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080252 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080258 2024-11-11T16:38:00,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,265 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1020, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,265 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080258) roll requested 2024-11-11T16:38:00,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080265 2024-11-11T16:38:00,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080258 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080265 2024-11-11T16:38:00,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1021, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080265) roll requested 2024-11-11T16:38:00,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080272 2024-11-11T16:38:00,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080265 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080272 2024-11-11T16:38:00,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,278 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,278 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1022, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,279 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080272) roll requested 2024-11-11T16:38:00,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080279 2024-11-11T16:38:00,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080279 2024-11-11T16:38:00,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1023, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080279) roll requested 2024-11-11T16:38:00,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080285 2024-11-11T16:38:00,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080279 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080285 2024-11-11T16:38:00,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,291 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,291 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1024, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080285) roll requested 2024-11-11T16:38:00,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080292 2024-11-11T16:38:00,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080292 2024-11-11T16:38:00,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,299 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,299 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1025, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,299 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080292) roll requested 2024-11-11T16:38:00,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080299 2024-11-11T16:38:00,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080299 2024-11-11T16:38:00,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,305 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1026, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080299) roll requested 2024-11-11T16:38:00,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080305 2024-11-11T16:38:00,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080305 2024-11-11T16:38:00,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1027, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080305) roll requested 2024-11-11T16:38:00,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080312 2024-11-11T16:38:00,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080312 2024-11-11T16:38:00,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,319 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1028, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,319 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080312) roll requested 2024-11-11T16:38:00,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080319 2024-11-11T16:38:00,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080319 2024-11-11T16:38:00,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1029, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,326 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080319) roll requested 2024-11-11T16:38:00,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080326 2024-11-11T16:38:00,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080319 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080326 2024-11-11T16:38:00,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,333 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1030, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080326) roll requested 2024-11-11T16:38:00,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080333 2024-11-11T16:38:00,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080326 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080333 2024-11-11T16:38:00,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,340 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1031, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,340 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080333) roll requested 2024-11-11T16:38:00,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080340 2024-11-11T16:38:00,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080340 2024-11-11T16:38:00,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,347 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,347 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1032, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,347 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080340) roll requested 2024-11-11T16:38:00,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080348 2024-11-11T16:38:00,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080340 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080348 2024-11-11T16:38:00,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,354 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,354 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1033, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,354 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080348) roll requested 2024-11-11T16:38:00,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080354 2024-11-11T16:38:00,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080354 2024-11-11T16:38:00,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,361 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,361 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1034, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080354) roll requested 2024-11-11T16:38:00,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080361 2024-11-11T16:38:00,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080354 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080361 2024-11-11T16:38:00,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,367 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,367 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1035, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,367 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080361) roll requested 2024-11-11T16:38:00,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080368 2024-11-11T16:38:00,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080368 2024-11-11T16:38:00,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,374 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1036, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,374 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080368) roll requested 2024-11-11T16:38:00,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080374 2024-11-11T16:38:00,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080374 2024-11-11T16:38:00,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1037, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080374) roll requested 2024-11-11T16:38:00,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080380 2024-11-11T16:38:00,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080374 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080380 2024-11-11T16:38:00,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1038, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,387 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080380) roll requested 2024-11-11T16:38:00,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080387 2024-11-11T16:38:00,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080387 2024-11-11T16:38:00,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,393 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,393 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1039, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,393 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080387) roll requested 2024-11-11T16:38:00,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080393 2024-11-11T16:38:00,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080393 2024-11-11T16:38:00,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,401 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1040, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080393) roll requested 2024-11-11T16:38:00,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080401 2024-11-11T16:38:00,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080393 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080401 2024-11-11T16:38:00,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,409 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,409 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1041, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,409 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080401) roll requested 2024-11-11T16:38:00,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080409 2024-11-11T16:38:00,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080409 2024-11-11T16:38:00,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1042, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080409) roll requested 2024-11-11T16:38:00,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080417 2024-11-11T16:38:00,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080417 2024-11-11T16:38:00,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1043, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080417) roll requested 2024-11-11T16:38:00,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080424 2024-11-11T16:38:00,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080424 2024-11-11T16:38:00,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1044, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,433 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080424) roll requested 2024-11-11T16:38:00,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080433 2024-11-11T16:38:00,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080433 2024-11-11T16:38:00,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1045, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,439 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080433) roll requested 2024-11-11T16:38:00,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080439 2024-11-11T16:38:00,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080439 2024-11-11T16:38:00,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,447 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,447 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1046, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,447 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080439) roll requested 2024-11-11T16:38:00,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080447 2024-11-11T16:38:00,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080439 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080447 2024-11-11T16:38:00,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,456 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1047, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080447) roll requested 2024-11-11T16:38:00,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080457 2024-11-11T16:38:00,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080447 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080457 2024-11-11T16:38:00,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1048, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080457) roll requested 2024-11-11T16:38:00,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080465 2024-11-11T16:38:00,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080465 2024-11-11T16:38:00,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,471 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,471 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1049, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,471 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080465) roll requested 2024-11-11T16:38:00,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080471 2024-11-11T16:38:00,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080471 2024-11-11T16:38:00,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,478 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1050, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080471) roll requested 2024-11-11T16:38:00,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080478 2024-11-11T16:38:00,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080471 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080478 2024-11-11T16:38:00,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1051, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080478) roll requested 2024-11-11T16:38:00,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080485 2024-11-11T16:38:00,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080485 2024-11-11T16:38:00,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,491 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,491 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1052, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,491 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080485) roll requested 2024-11-11T16:38:00,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080491 2024-11-11T16:38:00,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080491 2024-11-11T16:38:00,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1053, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080491) roll requested 2024-11-11T16:38:00,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080498 2024-11-11T16:38:00,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080498 2024-11-11T16:38:00,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,505 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,505 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1054, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080498) roll requested 2024-11-11T16:38:00,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080505 2024-11-11T16:38:00,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080505 2024-11-11T16:38:00,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1055, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080505) roll requested 2024-11-11T16:38:00,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080512 2024-11-11T16:38:00,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080512 2024-11-11T16:38:00,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1056, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080512) roll requested 2024-11-11T16:38:00,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080519 2024-11-11T16:38:00,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080512 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080519 2024-11-11T16:38:00,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1057, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080519) roll requested 2024-11-11T16:38:00,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080529 2024-11-11T16:38:00,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080519 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080529 2024-11-11T16:38:00,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,535 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1058, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,535 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080529) roll requested 2024-11-11T16:38:00,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080535 2024-11-11T16:38:00,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080535 2024-11-11T16:38:00,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1059, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080535) roll requested 2024-11-11T16:38:00,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080543 2024-11-11T16:38:00,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080535 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080543 2024-11-11T16:38:00,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,550 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1060, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080543) roll requested 2024-11-11T16:38:00,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080550 2024-11-11T16:38:00,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080550 2024-11-11T16:38:00,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,559 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,559 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1061, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,559 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080550) roll requested 2024-11-11T16:38:00,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080559 2024-11-11T16:38:00,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080559 2024-11-11T16:38:00,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,567 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,567 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1062, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,567 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080559) roll requested 2024-11-11T16:38:00,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080567 2024-11-11T16:38:00,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080559 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080567 2024-11-11T16:38:00,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1063, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,575 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080567) roll requested 2024-11-11T16:38:00,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080575 2024-11-11T16:38:00,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080567 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080575 2024-11-11T16:38:00,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,583 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,583 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1064, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,583 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080575) roll requested 2024-11-11T16:38:00,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080583 2024-11-11T16:38:00,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080575 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080583 2024-11-11T16:38:00,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1065, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080583) roll requested 2024-11-11T16:38:00,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080593 2024-11-11T16:38:00,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080583 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080593 2024-11-11T16:38:00,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,609 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1066, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080593) roll requested 2024-11-11T16:38:00,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080610 2024-11-11T16:38:00,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080610 2024-11-11T16:38:00,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,625 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,625 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1067, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080610) roll requested 2024-11-11T16:38:00,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080625 2024-11-11T16:38:00,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080625 2024-11-11T16:38:00,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1068, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080625) roll requested 2024-11-11T16:38:00,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080634 2024-11-11T16:38:00,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080634 2024-11-11T16:38:00,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,646 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1069, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080634) roll requested 2024-11-11T16:38:00,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080647 2024-11-11T16:38:00,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080634 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080647 2024-11-11T16:38:00,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,656 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1070, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,656 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080647) roll requested 2024-11-11T16:38:00,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080656 2024-11-11T16:38:00,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080656 2024-11-11T16:38:00,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1071, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080656) roll requested 2024-11-11T16:38:00,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080664 2024-11-11T16:38:00,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080656 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080664 2024-11-11T16:38:00,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,672 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1072, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,672 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080664) roll requested 2024-11-11T16:38:00,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080672 2024-11-11T16:38:00,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080672 2024-11-11T16:38:00,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,679 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,679 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1073, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,679 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080672) roll requested 2024-11-11T16:38:00,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080680 2024-11-11T16:38:00,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080672 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080680 2024-11-11T16:38:00,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1074, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080680) roll requested 2024-11-11T16:38:00,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080687 2024-11-11T16:38:00,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080680 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080687 2024-11-11T16:38:00,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1075, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080687) roll requested 2024-11-11T16:38:00,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080694 2024-11-11T16:38:00,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080694 2024-11-11T16:38:00,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,706 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,706 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1076, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,706 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080694) roll requested 2024-11-11T16:38:00,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080706 2024-11-11T16:38:00,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080706 2024-11-11T16:38:00,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,712 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,713 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1077, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,713 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080706) roll requested 2024-11-11T16:38:00,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080713 2024-11-11T16:38:00,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080706 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080713 2024-11-11T16:38:00,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,719 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,720 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1078, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,720 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080713) roll requested 2024-11-11T16:38:00,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080720 2024-11-11T16:38:00,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080713 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080720 2024-11-11T16:38:00,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1079, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080720) roll requested 2024-11-11T16:38:00,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080726 2024-11-11T16:38:00,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080720 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080726 2024-11-11T16:38:00,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1080, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080726) roll requested 2024-11-11T16:38:00,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080734 2024-11-11T16:38:00,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080734 2024-11-11T16:38:00,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,741 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1081, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080734) roll requested 2024-11-11T16:38:00,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080741 2024-11-11T16:38:00,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080741 2024-11-11T16:38:00,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1082, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080741) roll requested 2024-11-11T16:38:00,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080748 2024-11-11T16:38:00,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080748 2024-11-11T16:38:00,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1083, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,757 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080748) roll requested 2024-11-11T16:38:00,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080757 2024-11-11T16:38:00,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080757 2024-11-11T16:38:00,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,763 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1084, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,763 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080757) roll requested 2024-11-11T16:38:00,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080763 2024-11-11T16:38:00,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080763 2024-11-11T16:38:00,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,770 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,771 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1085, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,771 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080763) roll requested 2024-11-11T16:38:00,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080771 2024-11-11T16:38:00,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080763 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080771 2024-11-11T16:38:00,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,778 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1086, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,778 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080771) roll requested 2024-11-11T16:38:00,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080778 2024-11-11T16:38:00,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080771 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080778 2024-11-11T16:38:00,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,789 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1087, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,789 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080778) roll requested 2024-11-11T16:38:00,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080789 2024-11-11T16:38:00,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080778 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080789 2024-11-11T16:38:00,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,797 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,797 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1088, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,797 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080789) roll requested 2024-11-11T16:38:00,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080797 2024-11-11T16:38:00,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080797 2024-11-11T16:38:00,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,804 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,804 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1089, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080797) roll requested 2024-11-11T16:38:00,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080805 2024-11-11T16:38:00,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080797 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080805 2024-11-11T16:38:00,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,812 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1090, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,812 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080805) roll requested 2024-11-11T16:38:00,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080812 2024-11-11T16:38:00,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080812 2024-11-11T16:38:00,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,819 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1091, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,820 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080812) roll requested 2024-11-11T16:38:00,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080820 2024-11-11T16:38:00,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080812 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080820 2024-11-11T16:38:00,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,829 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1092, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,829 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080820) roll requested 2024-11-11T16:38:00,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080829 2024-11-11T16:38:00,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080820 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080829 2024-11-11T16:38:00,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,837 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1093, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080829) roll requested 2024-11-11T16:38:00,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080837 2024-11-11T16:38:00,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080837 2024-11-11T16:38:00,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,844 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,844 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1094, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,844 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080837) roll requested 2024-11-11T16:38:00,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080844 2024-11-11T16:38:00,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080844 2024-11-11T16:38:00,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,851 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,852 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1095, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,852 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080844) roll requested 2024-11-11T16:38:00,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080852 2024-11-11T16:38:00,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080844 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080852 2024-11-11T16:38:00,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,858 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1096, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,859 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080852) roll requested 2024-11-11T16:38:00,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080859 2024-11-11T16:38:00,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080859 2024-11-11T16:38:00,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,866 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1097, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,866 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080859) roll requested 2024-11-11T16:38:00,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080866 2024-11-11T16:38:00,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080859 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080866 2024-11-11T16:38:00,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1098, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080866) roll requested 2024-11-11T16:38:00,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080873 2024-11-11T16:38:00,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080866 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080873 2024-11-11T16:38:00,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1099, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080873) roll requested 2024-11-11T16:38:00,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080880 2024-11-11T16:38:00,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080880 2024-11-11T16:38:00,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,886 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,886 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1100, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,887 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080880) roll requested 2024-11-11T16:38:00,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080887 2024-11-11T16:38:00,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080887 2024-11-11T16:38:00,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1101, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080887) roll requested 2024-11-11T16:38:00,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080894 2024-11-11T16:38:00,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080894 2024-11-11T16:38:00,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1102, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080894) roll requested 2024-11-11T16:38:00,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080900 2024-11-11T16:38:00,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080900 2024-11-11T16:38:00,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1103, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080900) roll requested 2024-11-11T16:38:00,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080907 2024-11-11T16:38:00,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080907 2024-11-11T16:38:00,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1104, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080907) roll requested 2024-11-11T16:38:00,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080915 2024-11-11T16:38:00,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080915 2024-11-11T16:38:00,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1105, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080915) roll requested 2024-11-11T16:38:00,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080925 2024-11-11T16:38:00,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080925 2024-11-11T16:38:00,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1106, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,934 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080925) roll requested 2024-11-11T16:38:00,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080934 2024-11-11T16:38:00,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080934 2024-11-11T16:38:00,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,942 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,942 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1107, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,942 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080934) roll requested 2024-11-11T16:38:00,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080942 2024-11-11T16:38:00,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080942 2024-11-11T16:38:00,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1108, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080942) roll requested 2024-11-11T16:38:00,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080952 2024-11-11T16:38:00,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080942 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080952 2024-11-11T16:38:00,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1109, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,961 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080952) roll requested 2024-11-11T16:38:00,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080961 2024-11-11T16:38:00,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080961 2024-11-11T16:38:00,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1110, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080961) roll requested 2024-11-11T16:38:00,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080972 2024-11-11T16:38:00,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080961 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080972 2024-11-11T16:38:00,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,980 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,980 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1111, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,981 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080972) roll requested 2024-11-11T16:38:00,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080981 2024-11-11T16:38:00,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080981 2024-11-11T16:38:00,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,987 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,987 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1112, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,987 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080981) roll requested 2024-11-11T16:38:00,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080987 2024-11-11T16:38:00,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:00,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080981 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080987 2024-11-11T16:38:00,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:00,998 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:00,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1113, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:00,999 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:00,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080987) roll requested 2024-11-11T16:38:00,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343080999 2024-11-11T16:38:01,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080999 2024-11-11T16:38:01,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,005 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1114, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343080999) roll requested 2024-11-11T16:38:01,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081005 2024-11-11T16:38:01,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343080999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081005 2024-11-11T16:38:01,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,012 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1115, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081005) roll requested 2024-11-11T16:38:01,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081012 2024-11-11T16:38:01,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081012 2024-11-11T16:38:01,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,019 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1116, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081012) roll requested 2024-11-11T16:38:01,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081019 2024-11-11T16:38:01,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081019 2024-11-11T16:38:01,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,026 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1117, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,026 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081019) roll requested 2024-11-11T16:38:01,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081026 2024-11-11T16:38:01,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081026 2024-11-11T16:38:01,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1118, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081026) roll requested 2024-11-11T16:38:01,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081033 2024-11-11T16:38:01,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081033 2024-11-11T16:38:01,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,040 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1119, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,040 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081033) roll requested 2024-11-11T16:38:01,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081040 2024-11-11T16:38:01,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081040 2024-11-11T16:38:01,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,046 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,047 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1120, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,047 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081040) roll requested 2024-11-11T16:38:01,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081047 2024-11-11T16:38:01,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081040 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081047 2024-11-11T16:38:01,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1121, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081047) roll requested 2024-11-11T16:38:01,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081054 2024-11-11T16:38:01,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081047 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081054 2024-11-11T16:38:01,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1122, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081054) roll requested 2024-11-11T16:38:01,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081061 2024-11-11T16:38:01,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081061 2024-11-11T16:38:01,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,067 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1123, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,067 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081061) roll requested 2024-11-11T16:38:01,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081068 2024-11-11T16:38:01,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081068 2024-11-11T16:38:01,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,074 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,074 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1124, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,074 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081068) roll requested 2024-11-11T16:38:01,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081074 2024-11-11T16:38:01,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081074 2024-11-11T16:38:01,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,081 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1125, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,081 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081074) roll requested 2024-11-11T16:38:01,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081081 2024-11-11T16:38:01,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081074 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081081 2024-11-11T16:38:01,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,087 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,087 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1126, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,087 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081081) roll requested 2024-11-11T16:38:01,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081087 2024-11-11T16:38:01,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081081 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081087 2024-11-11T16:38:01,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1127, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081087) roll requested 2024-11-11T16:38:01,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081097 2024-11-11T16:38:01,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081087 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081097 2024-11-11T16:38:01,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1128, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081097) roll requested 2024-11-11T16:38:01,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081105 2024-11-11T16:38:01,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081105 2024-11-11T16:38:01,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,111 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1129, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,112 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081105) roll requested 2024-11-11T16:38:01,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081112 2024-11-11T16:38:01,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081112 2024-11-11T16:38:01,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,118 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,119 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1130, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081112) roll requested 2024-11-11T16:38:01,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081119 2024-11-11T16:38:01,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081112 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081119 2024-11-11T16:38:01,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,126 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1131, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081119) roll requested 2024-11-11T16:38:01,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081126 2024-11-11T16:38:01,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081126 2024-11-11T16:38:01,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1132, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081126) roll requested 2024-11-11T16:38:01,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081134 2024-11-11T16:38:01,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081134 2024-11-11T16:38:01,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,141 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1133, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081134) roll requested 2024-11-11T16:38:01,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081141 2024-11-11T16:38:01,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081134 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081141 2024-11-11T16:38:01,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,150 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1134, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,150 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081141) roll requested 2024-11-11T16:38:01,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081150 2024-11-11T16:38:01,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081150 2024-11-11T16:38:01,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,156 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1135, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081150) roll requested 2024-11-11T16:38:01,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081156 2024-11-11T16:38:01,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081150 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081156 2024-11-11T16:38:01,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,163 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,163 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1136, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,163 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081156) roll requested 2024-11-11T16:38:01,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081164 2024-11-11T16:38:01,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081164 2024-11-11T16:38:01,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,171 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,171 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1137, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,171 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081164) roll requested 2024-11-11T16:38:01,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081171 2024-11-11T16:38:01,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081164 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081171 2024-11-11T16:38:01,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,179 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,179 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1138, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081171) roll requested 2024-11-11T16:38:01,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081180 2024-11-11T16:38:01,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081171 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081180 2024-11-11T16:38:01,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,186 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,186 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1139, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,186 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081180) roll requested 2024-11-11T16:38:01,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081186 2024-11-11T16:38:01,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081186 2024-11-11T16:38:01,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,193 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,193 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1140, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,193 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081186) roll requested 2024-11-11T16:38:01,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081193 2024-11-11T16:38:01,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081186 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081193 2024-11-11T16:38:01,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,200 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,200 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1141, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,200 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081193) roll requested 2024-11-11T16:38:01,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081201 2024-11-11T16:38:01,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081193 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081201 2024-11-11T16:38:01,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,207 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,207 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1142, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,207 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081201) roll requested 2024-11-11T16:38:01,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081207 2024-11-11T16:38:01,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081207 2024-11-11T16:38:01,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,214 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,214 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1143, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,214 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081207) roll requested 2024-11-11T16:38:01,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081214 2024-11-11T16:38:01,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081207 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081214 2024-11-11T16:38:01,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,221 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,221 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1144, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,222 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081214) roll requested 2024-11-11T16:38:01,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081222 2024-11-11T16:38:01,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081214 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081222 2024-11-11T16:38:01,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,228 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,228 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1145, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,228 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081222) roll requested 2024-11-11T16:38:01,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081228 2024-11-11T16:38:01,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081222 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081228 2024-11-11T16:38:01,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,235 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,235 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1146, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,235 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081228) roll requested 2024-11-11T16:38:01,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081235 2024-11-11T16:38:01,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081228 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081235 2024-11-11T16:38:01,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,242 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,242 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1147, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,243 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081235) roll requested 2024-11-11T16:38:01,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081243 2024-11-11T16:38:01,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081235 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081243 2024-11-11T16:38:01,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,249 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,250 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1148, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,250 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081243) roll requested 2024-11-11T16:38:01,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081250 2024-11-11T16:38:01,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081243 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081250 2024-11-11T16:38:01,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1149, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,256 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081250) roll requested 2024-11-11T16:38:01,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081256 2024-11-11T16:38:01,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081250 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081256 2024-11-11T16:38:01,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,263 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1150, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081256) roll requested 2024-11-11T16:38:01,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081264 2024-11-11T16:38:01,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081256 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081264 2024-11-11T16:38:01,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,273 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,273 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1151, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,273 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081264) roll requested 2024-11-11T16:38:01,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081273 2024-11-11T16:38:01,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081273 2024-11-11T16:38:01,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,280 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1152, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081273) roll requested 2024-11-11T16:38:01,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081280 2024-11-11T16:38:01,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081280 2024-11-11T16:38:01,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,286 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,286 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1153, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,287 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081280) roll requested 2024-11-11T16:38:01,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081287 2024-11-11T16:38:01,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081287 2024-11-11T16:38:01,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1154, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,293 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081287) roll requested 2024-11-11T16:38:01,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081293 2024-11-11T16:38:01,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081287 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081293 2024-11-11T16:38:01,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,300 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1155, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,300 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081293) roll requested 2024-11-11T16:38:01,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081300 2024-11-11T16:38:01,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081300 2024-11-11T16:38:01,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,307 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,307 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1156, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,307 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081300) roll requested 2024-11-11T16:38:01,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081307 2024-11-11T16:38:01,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081300 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081307 2024-11-11T16:38:01,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,314 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1157, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,314 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081307) roll requested 2024-11-11T16:38:01,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081314 2024-11-11T16:38:01,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081307 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081314 2024-11-11T16:38:01,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,321 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,321 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1158, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081314) roll requested 2024-11-11T16:38:01,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081321 2024-11-11T16:38:01,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081314 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081321 2024-11-11T16:38:01,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,327 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1159, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081321) roll requested 2024-11-11T16:38:01,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081328 2024-11-11T16:38:01,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081328 2024-11-11T16:38:01,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1160, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,334 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081328) roll requested 2024-11-11T16:38:01,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081334 2024-11-11T16:38:01,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081328 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081334 2024-11-11T16:38:01,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,341 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1161, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,341 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081334) roll requested 2024-11-11T16:38:01,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081341 2024-11-11T16:38:01,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081341 2024-11-11T16:38:01,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,347 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,347 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1162, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,347 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081341) roll requested 2024-11-11T16:38:01,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081347 2024-11-11T16:38:01,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081347 2024-11-11T16:38:01,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,354 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,354 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1163, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,354 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081347) roll requested 2024-11-11T16:38:01,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081354 2024-11-11T16:38:01,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081347 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081354 2024-11-11T16:38:01,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,361 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,361 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1164, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081354) roll requested 2024-11-11T16:38:01,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081361 2024-11-11T16:38:01,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081354 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081361 2024-11-11T16:38:01,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,367 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1165, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081361) roll requested 2024-11-11T16:38:01,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081368 2024-11-11T16:38:01,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081368 2024-11-11T16:38:01,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,375 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,375 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1166, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,375 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081368) roll requested 2024-11-11T16:38:01,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081375 2024-11-11T16:38:01,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081375 2024-11-11T16:38:01,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,383 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,383 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1167, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,383 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081375) roll requested 2024-11-11T16:38:01,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081383 2024-11-11T16:38:01,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081383 2024-11-11T16:38:01,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,391 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1168, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,391 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081383) roll requested 2024-11-11T16:38:01,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081391 2024-11-11T16:38:01,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081383 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081391 2024-11-11T16:38:01,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,398 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,398 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1169, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,398 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081391) roll requested 2024-11-11T16:38:01,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081399 2024-11-11T16:38:01,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081399 2024-11-11T16:38:01,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1170, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,405 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081399) roll requested 2024-11-11T16:38:01,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081405 2024-11-11T16:38:01,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081399 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081405 2024-11-11T16:38:01,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,412 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,413 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1171, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,413 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081405) roll requested 2024-11-11T16:38:01,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081413 2024-11-11T16:38:01,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081413 2024-11-11T16:38:01,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1172, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081413) roll requested 2024-11-11T16:38:01,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081420 2024-11-11T16:38:01,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081413 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081420 2024-11-11T16:38:01,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,427 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1173, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,427 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081420) roll requested 2024-11-11T16:38:01,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081428 2024-11-11T16:38:01,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081428 2024-11-11T16:38:01,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,434 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,434 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1174, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,434 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081428) roll requested 2024-11-11T16:38:01,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081434 2024-11-11T16:38:01,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081434 2024-11-11T16:38:01,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,442 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,442 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1175, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081434) roll requested 2024-11-11T16:38:01,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081442 2024-11-11T16:38:01,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081434 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081442 2024-11-11T16:38:01,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1176, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081442) roll requested 2024-11-11T16:38:01,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081450 2024-11-11T16:38:01,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081450 2024-11-11T16:38:01,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,456 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1177, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081450) roll requested 2024-11-11T16:38:01,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081457 2024-11-11T16:38:01,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081457 2024-11-11T16:38:01,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,463 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1178, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081457) roll requested 2024-11-11T16:38:01,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081464 2024-11-11T16:38:01,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081464 2024-11-11T16:38:01,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,470 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,470 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1179, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,471 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081464) roll requested 2024-11-11T16:38:01,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081471 2024-11-11T16:38:01,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081471 2024-11-11T16:38:01,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1180, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081471) roll requested 2024-11-11T16:38:01,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081477 2024-11-11T16:38:01,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081471 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081477 2024-11-11T16:38:01,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1181, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081477) roll requested 2024-11-11T16:38:01,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081485 2024-11-11T16:38:01,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081485 2024-11-11T16:38:01,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,500 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,500 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1182, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,500 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081485) roll requested 2024-11-11T16:38:01,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081500 2024-11-11T16:38:01,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081500 2024-11-11T16:38:01,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,507 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1183, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081500) roll requested 2024-11-11T16:38:01,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081508 2024-11-11T16:38:01,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081500 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081508 2024-11-11T16:38:01,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1184, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081508) roll requested 2024-11-11T16:38:01,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081519 2024-11-11T16:38:01,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081508 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081519 2024-11-11T16:38:01,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,526 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,526 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1185, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,526 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081519) roll requested 2024-11-11T16:38:01,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081526 2024-11-11T16:38:01,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081519 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081526 2024-11-11T16:38:01,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,533 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,533 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1186, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,533 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081526) roll requested 2024-11-11T16:38:01,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081533 2024-11-11T16:38:01,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081526 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081533 2024-11-11T16:38:01,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,540 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,540 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1187, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,540 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081533) roll requested 2024-11-11T16:38:01,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081540 2024-11-11T16:38:01,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081533 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081540 2024-11-11T16:38:01,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-11T16:38:01,547 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1188, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,547 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731343081540) roll requested 2024-11-11T16:38:01,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731343081547 2024-11-11T16:38:01,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081540 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731343081547 2024-11-11T16:38:01,557 ERROR [Time-limited test {}] wal.AbstractFSWAL(939): Failed log archiving for the log /a/b, java.io.IOException: throw archival exception at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archiveLogFile(TestFailedAppendAndSync.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archive(AbstractFSWAL.java:934) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archive(TestFailedAppendAndSync.java:143) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync.testLockupAroundBadAssignSync(TestFailedAppendAndSync.java:280) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:38:01,559 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 682720dead0ade45c99f3f8b05a66179, disabling compactions & flushes 2024-11-11T16:38:01,559 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testLockupAroundBadAssignSync,,1731343051198.682720dead0ade45c99f3f8b05a66179. 2024-11-11T16:38:01,559 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testLockupAroundBadAssignSync,,1731343051198.682720dead0ade45c99f3f8b05a66179. 2024-11-11T16:38:01,560 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testLockupAroundBadAssignSync,,1731343051198.682720dead0ade45c99f3f8b05a66179. after waiting 0 ms 2024-11-11T16:38:01,560 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testLockupAroundBadAssignSync,,1731343051198.682720dead0ade45c99f3f8b05a66179. 2024-11-11T16:38:01,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-11T16:38:01,561 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1189, max=55; forcing (partial) flush of 1 region(s): 682720dead0ade45c99f3f8b05a66179[MyCF] 2024-11-11T16:38:01,561 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 682720dead0ade45c99f3f8b05a66179, because it is not online on us 2024-11-11T16:38:01,561 INFO [LogRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T16:38:01,561 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 83 in region testLockupAroundBadAssignSync,,1731343051198.682720dead0ade45c99f3f8b05a66179. 2024-11-11T16:38:01,561 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testLockupAroundBadAssignSync,,1731343051198.682720dead0ade45c99f3f8b05a66179. 2024-11-11T16:38:01,561 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 682720dead0ade45c99f3f8b05a66179: Waiting for close lock at 1731343081559Disabling compacts and flushes for region at 1731343081559Disabling writes for close at 1731343081560 (+1 ms)Writing region close event to WAL at 1731343081561 (+1 ms)Closed at 1731343081561 2024-11-11T16:38:01,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-11T16:38:01,893 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: DodgyFSLog wal:(num 1731343081547) 2024-11-11T16:38:01,894 INFO [Time-limited test {}] regionserver.TestFailedAppendAndSync(105): Cleaning test directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f6776eb5-1084-6444-7c8d-c2790b22b040 2024-11-11T16:38:02,459 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=22 (was 11) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=287 (was 287), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=376 (was 439), ProcessCount=11 (was 11), AvailableMemoryMB=2327 (was 3045)